diff --git a/DEPS.bzl b/DEPS.bzl index efe2f8115232a..804e8fd6cb029 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -5971,13 +5971,13 @@ def go_deps(): name = "com_github_pingcap_tipb", build_file_proto_mode = "disable_global", importpath = "github.com/pingcap/tipb", - sha256 = "af0d5b3d3d28d8c7e43b446873541c0f53a1c552e78fd2eda33931302c9f62e3", - strip_prefix = "github.com/pingcap/tipb@v0.0.0-20260507102040-d3d6e146648f", + sha256 = "8fc936a7308e69dd6a95e5594576ccf3f6229bcb9ef90804c52fb065bb09b403", + strip_prefix = "github.com/pingcap/tipb@v0.0.0-20260605083900-f9f651ef5fbc", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260507102040-d3d6e146648f.zip", - "http://ats.apps.svc/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260507102040-d3d6e146648f.zip", - "https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260507102040-d3d6e146648f.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260507102040-d3d6e146648f.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260605083900-f9f651ef5fbc.zip", + "http://ats.apps.svc/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260605083900-f9f651ef5fbc.zip", + "https://cache.hawkingrei.com/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260605083900-f9f651ef5fbc.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/pingcap/tipb/com_github_pingcap_tipb-v0.0.0-20260605083900-f9f651ef5fbc.zip", ], ) go_repository( diff --git a/go.mod b/go.mod index dbeb33b9b1ec6..f4d6ff153f1fb 100644 --- a/go.mod +++ b/go.mod @@ -100,7 +100,7 @@ require ( github.com/pingcap/log v1.1.1-0.20250917021125-19901e015dc9 github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 github.com/pingcap/tidb/pkg/parser v0.0.0-20211011031125-9b13dc409c5e - github.com/pingcap/tipb v0.0.0-20260507102040-d3d6e146648f + github.com/pingcap/tipb v0.0.0-20260605083900-f9f651ef5fbc github.com/prometheus/client_golang v1.23.0 github.com/prometheus/client_model v0.6.2 github.com/prometheus/common v0.65.0 diff --git a/go.sum b/go.sum index a8bb5323cf8f4..79a5622331376 100644 --- a/go.sum +++ b/go.sum @@ -663,8 +663,8 @@ github.com/pingcap/log v1.1.1-0.20250917021125-19901e015dc9 h1:qG9BSvlWFEE5otQGa github.com/pingcap/log v1.1.1-0.20250917021125-19901e015dc9/go.mod h1:ORfBOFp1eteu2odzsyaxI+b8TzJwgjwyQcGhI+9SfEA= github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 h1:T4pXRhBflzDeAhmOQHNPRRogMYxP13V7BkYw3ZsoSfE= github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5/go.mod h1:rlimy0GcTvjiJqvD5mXTRr8O2eNZPBrcUgiWVYp9530= -github.com/pingcap/tipb v0.0.0-20260507102040-d3d6e146648f h1:ld8bQ5d0zh1B0HRbJHiaf2seZvcVV5Ug2rih70uNJMM= -github.com/pingcap/tipb v0.0.0-20260507102040-d3d6e146648f/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= +github.com/pingcap/tipb v0.0.0-20260605083900-f9f651ef5fbc h1:wxolKysltFSu8gxWJBdUdWuTBoSuY3MjNIIZI5S9JLY= +github.com/pingcap/tipb v0.0.0-20260605083900-f9f651ef5fbc/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= diff --git a/pkg/planner/core/casetest/rule/BUILD.bazel b/pkg/planner/core/casetest/rule/BUILD.bazel index 05152fed4cd66..dc805c9acd0a0 100644 --- a/pkg/planner/core/casetest/rule/BUILD.bazel +++ b/pkg/planner/core/casetest/rule/BUILD.bazel @@ -15,7 +15,7 @@ go_test( ], data = glob(["testdata/**"]), flaky = True, - shard_count = 12, + shard_count = 13, deps = [ "//pkg/config", "//pkg/domain", diff --git a/pkg/planner/core/exhaust_physical_plans.go b/pkg/planner/core/exhaust_physical_plans.go index baa7b6d49f008..1d869556aa667 100644 --- a/pkg/planner/core/exhaust_physical_plans.go +++ b/pkg/planner/core/exhaust_physical_plans.go @@ -2266,7 +2266,10 @@ func getPhysTopN(lt *logicalop.LogicalTopN, prop *property.PhysicalProperty) []b if mppAllowed { allTaskTypes = append(allTaskTypes, property.MppTaskType) } - ret := make([]base.PhysicalPlan, 0, len(allTaskTypes)) + ret := make([]base.PhysicalPlan, 0, len(allTaskTypes)+1) + if canUsePartialOrder4TopN(lt) { + ret = append(ret, getPhysTopNWithPartialOrderProperty(lt, prop)...) + } for _, tp := range allTaskTypes { resultProp := &property.PhysicalProperty{TaskTp: tp, ExpectedCnt: math.MaxFloat64, CTEProducerStatus: prop.CTEProducerStatus} topN := PhysicalTopN{ @@ -2318,6 +2321,56 @@ func getPhysTopN(lt *logicalop.LogicalTopN, prop *property.PhysicalProperty) []b return ret } +// canUsePartialOrder4TopN checks if the TopN's child tree satisfies the supported partial-order pattern. +func canUsePartialOrder4TopN(lt *logicalop.LogicalTopN) bool { + if !lt.SCtx().GetSessionVars().IsPartialOrderedIndexForTopNEnabled() { + return false + } + if len(lt.ByItems) == 0 { + return false + } + return checkPartialOrderPattern(lt.Children()[0]) +} + +func checkPartialOrderPattern(plan base.LogicalPlan) bool { + switch p := plan.(type) { + case *logicalop.DataSource: + return true + case *logicalop.LogicalSelection: + return len(p.Children()) == 1 && checkPartialOrderPattern(p.Children()[0]) + case *logicalop.LogicalProjection: + return len(p.Children()) == 1 && checkPartialOrderPattern(p.Children()[0]) + default: + return false + } +} + +func getPhysTopNWithPartialOrderProperty(lt *logicalop.LogicalTopN, prop *property.PhysicalProperty) []base.PhysicalPlan { + sortItems := make([]*property.SortItem, 0, len(lt.ByItems)) + for _, byItem := range lt.ByItems { + col, ok := byItem.Expr.(*expression.Column) + if !ok { + return nil + } + sortItems = append(sortItems, &property.SortItem{Col: col, Desc: byItem.Desc}) + } + partialOrderProp := &property.PhysicalProperty{ + TaskTp: property.CopMultiReadTaskType, + ExpectedCnt: math.MaxFloat64, + PartialOrderInfo: &property.PartialOrderInfo{ + SortItems: sortItems, + }, + CTEProducerStatus: prop.CTEProducerStatus, + } + topN := PhysicalTopN{ + ByItems: lt.ByItems, + PartitionBy: lt.PartitionBy, + Count: lt.Count, + Offset: lt.Offset, + }.Init(lt.SCtx(), lt.StatsInfo(), lt.QueryBlockOffset(), partialOrderProp) + return []base.PhysicalPlan{topN} +} + func getPhysLimits(lt *logicalop.LogicalTopN, prop *property.PhysicalProperty) []base.PhysicalPlan { p, canPass := GetPropByOrderByItems(lt.ByItems) if !canPass { diff --git a/pkg/planner/core/explain.go b/pkg/planner/core/explain.go index b14e368b19c97..137a88abfe666 100644 --- a/pkg/planner/core/explain.go +++ b/pkg/planner/core/explain.go @@ -483,10 +483,21 @@ func (p *PhysicalLimit) ExplainInfo() string { } if redact == perrors.RedactLogDisable { fmt.Fprintf(buffer, "offset:%v, count:%v", p.Offset, p.Count) + if p.PrefixCol != nil { + fmt.Fprintf(buffer, ", prefix_col:%v, prefix_len:%v", + p.PrefixCol.ColumnExplainInfo(ectx, false), p.PrefixLen) + } } else if redact == perrors.RedactLogMarker { fmt.Fprintf(buffer, "offset:‹%v›, count:‹%v›", p.Offset, p.Count) + if p.PrefixCol != nil { + fmt.Fprintf(buffer, ", prefix_col:‹%v›, prefix_len:‹%v›", + p.PrefixCol.ColumnExplainInfo(ectx, false), p.PrefixLen) + } } else if redact == perrors.RedactLogEnable { fmt.Fprintf(buffer, "offset:?, count:?") + if p.PrefixCol != nil { + fmt.Fprintf(buffer, ", prefix_col:?, prefix_len:?") + } } return buffer.String() } @@ -790,10 +801,21 @@ func (p *PhysicalTopN) ExplainInfo() string { switch p.SCtx().GetSessionVars().EnableRedactLog { case perrors.RedactLogDisable: fmt.Fprintf(buffer, ", offset:%v, count:%v", p.Offset, p.Count) + if p.PrefixCol != nil { + fmt.Fprintf(buffer, ", prefix_col:%v, prefix_len:%v", + p.PrefixCol.ColumnExplainInfo(ectx, false), p.PrefixLen) + } case perrors.RedactLogMarker: fmt.Fprintf(buffer, ", offset:‹%v›, count:‹%v›", p.Offset, p.Count) + if p.PrefixCol != nil { + fmt.Fprintf(buffer, ", prefix_col:‹%v›, prefix_len:‹%v›", + p.PrefixCol.ColumnExplainInfo(ectx, false), p.PrefixLen) + } case perrors.RedactLogEnable: fmt.Fprintf(buffer, ", offset:?, count:?") + if p.PrefixCol != nil { + fmt.Fprintf(buffer, ", prefix_col:?, prefix_len:?") + } } return buffer.String() } diff --git a/pkg/planner/core/find_best_task.go b/pkg/planner/core/find_best_task.go index 5f51548ec8126..34b5d02d60763 100644 --- a/pkg/planner/core/find_best_task.go +++ b/pkg/planner/core/find_best_task.go @@ -683,9 +683,11 @@ type candidatePath struct { accessCondsColMap util.Col2Len // accessCondsColMap maps Column.UniqueID to column length for the columns in AccessConds. indexCondsColMap util.Col2Len // indexCondsColMap maps Column.UniqueID to column length for the columns in AccessConds and indexFilters. matchPropResult property.PhysicalPropMatchResult - indexJoinCols int // how many index columns are used in access conditions in this IndexJoin. - isFullRange bool // cached result of whether this path covers the full scan range. - eqOrInCount int // cached result of equalPredicateCount(). + // partialOrderMatchResult records whether this path can provide partial order using prefix index. + partialOrderMatchResult property.PartialOrderMatchResult + indexJoinCols int // how many index columns are used in access conditions in this IndexJoin. + isFullRange bool // cached result of whether this path covers the full scan range. + eqOrInCount int // cached result of equalPredicateCount(). } func compareBool(l, r bool) int { @@ -1014,6 +1016,62 @@ func matchProperty(ds *logicalop.DataSource, path *util.AccessPath, prop *proper return matchResult } +// matchPartialOrderProperty checks if the index can provide partial order for TopN optimization. +// Unlike matchProperty, this function allows a prefix index to match ORDER BY columns. +func matchPartialOrderProperty(path *util.AccessPath, partialOrderInfo *property.PartialOrderInfo) property.PartialOrderMatchResult { + emptyResult := property.PartialOrderMatchResult{Matched: false} + if partialOrderInfo == nil || path.Index == nil || len(path.IdxCols) == 0 { + return emptyResult + } + + sortItems := partialOrderInfo.SortItems + if len(sortItems) == 0 { + return emptyResult + } + + allSameOrder, _ := partialOrderInfo.AllSameOrder() + if !allSameOrder { + return emptyResult + } + + indexColCount := len(path.Index.Columns) + if len(path.FullIdxCols) < indexColCount || len(path.FullIdxColLens) < indexColCount { + return emptyResult + } + if indexColCount > len(sortItems) { + return emptyResult + } + if path.Index.Columns[indexColCount-1].Length == types.UnspecifiedLength { + return emptyResult + } + + orderByCols := make([]*expression.Column, 0, len(sortItems)) + for _, item := range sortItems { + orderByCols = append(orderByCols, item.Col) + } + + for i := range indexColCount { + idxCol := path.FullIdxCols[i] + if idxCol == nil { + return emptyResult + } + if !orderByCols[i].EqualColumn(idxCol) { + return emptyResult + } + if path.FullIdxColLens[i] != types.UnspecifiedLength { + if i != indexColCount-1 { + return emptyResult + } + return property.PartialOrderMatchResult{ + Matched: true, + PrefixCol: idxCol, + PrefixLen: path.FullIdxColLens[i], + } + } + } + return emptyResult +} + // GroupRangesByCols groups the ranges by the values of the columns specified by groupByColIdxs. func GroupRangesByCols(ranges []*ranger.Range, groupByColIdxs []int) ([][]*ranger.Range, error) { groups := make(map[string][]*ranger.Range) @@ -1262,6 +1320,9 @@ func getTableCandidate(ds *logicalop.DataSource, path *util.AccessPath, prop *pr func getIndexCandidate(ds *logicalop.DataSource, path *util.AccessPath, prop *property.PhysicalProperty) *candidatePath { candidate := &candidatePath{path: path} candidate.matchPropResult = matchProperty(ds, path, prop) + if ds.SCtx().GetSessionVars().IsPartialOrderedIndexForTopNEnabled() && prop.PartialOrderInfo != nil { + candidate.partialOrderMatchResult = matchPartialOrderProperty(path, prop.PartialOrderInfo) + } candidate.accessCondsColMap = util.ExtractCol2Len(ds.SCtx().GetExprCtx().GetEvalCtx(), path.AccessConds, path.IdxCols, path.IdxColLens) candidate.indexCondsColMap = util.ExtractCol2Len(ds.SCtx().GetExprCtx().GetEvalCtx(), append(path.AccessConds, path.IndexFilters...), path.FullIdxCols, path.FullIdxColLens) candidate.isFullRange = path.IsFullScanRange(ds.TableInfo) @@ -1337,9 +1398,24 @@ func skylinePruning(ds *logicalop.DataSource, prop *property.PhysicalProperty) [ } var currentCandidate *candidatePath if path.IsTablePath() { + if prop.PartialOrderInfo != nil { + continue + } currentCandidate = getTableCandidate(ds, path, prop) } else { - if !(len(path.AccessConds) > 0 || !prop.IsSortItemEmpty() || path.Forced || path.IsSingleScan) { + var matchPartialOrderIndex bool + if ds.SCtx().GetSessionVars().IsPartialOrderedIndexForTopNEnabled() && + prop.PartialOrderInfo != nil { + if !matchPartialOrderProperty(path, prop.PartialOrderInfo).Matched { + continue + } + matchPartialOrderIndex = true + if path.Forced && !path.ForceNoKeepOrder { + path.ForcePartialOrder = true + } + } + keepIndex := len(path.AccessConds) > 0 || !prop.IsSortItemEmpty() || path.Forced || path.IsSingleScan || matchPartialOrderIndex + if !keepIndex { continue } // We will use index to generate physical plan if any of the following conditions is satisfied: @@ -1347,6 +1423,7 @@ func skylinePruning(ds *logicalop.DataSource, prop *property.PhysicalProperty) [ // 2. We have a non-empty prop to match. // 3. This index is forced to choose. // 4. The needed columns are all covered by index columns(and handleCol). + // 5. It matches PartialOrderInfo physical property for partial order optimization. currentCandidate = getIndexCandidate(ds, path, prop) } pruned := false @@ -2314,11 +2391,15 @@ func convertToIndexScan(ds *logicalop.DataSource, prop *property.PhysicalPropert return base.InvalidTask, nil } // If we need to keep order for the index scan, we should forbid the non-keep-order index scan when we try to generate the path. - if prop.IsSortItemEmpty() && candidate.path.ForceKeepOrder { + if !prop.NeedKeepOrder() && candidate.path.ForceKeepOrder { return base.InvalidTask, nil } // If we don't need to keep order for the index scan, we should forbid the non-keep-order index scan when we try to generate the path. - if !prop.IsSortItemEmpty() && candidate.path.ForceNoKeepOrder { + if prop.NeedKeepOrder() && candidate.path.ForceNoKeepOrder { + return base.InvalidTask, nil + } + // If partial order is forced, reject normal full-order or no-order candidates. + if candidate.path.ForcePartialOrder && prop.PartialOrderInfo == nil { return base.InvalidTask, nil } path := candidate.path @@ -2329,6 +2410,9 @@ func convertToIndexScan(ds *logicalop.DataSource, prop *property.PhysicalPropert tblCols: ds.TblCols, expectCnt: uint64(prop.ExpectedCnt), } + if candidate.partialOrderMatchResult.Matched { + cop.partialOrderMatchResult = &candidate.partialOrderMatchResult + } cop.physPlanPartInfo = &PhysPlanPartInfo{ PruningConds: pushDownNot(ds.SCtx().GetExprCtx(), ds.AllConds), PartitionNames: ds.PartitionNames, @@ -2372,7 +2456,7 @@ func convertToIndexScan(ds *logicalop.DataSource, prop *property.PhysicalPropert } } } - if candidate.matchPropResult.Matched() { + if prop.NeedKeepOrder() { cop.keepOrder = true if cop.tablePlan != nil && !ds.TableInfo.IsCommonHandle { col, isNew := cop.tablePlan.(*PhysicalTableScan).appendExtraHandleCol(ds) @@ -2387,8 +2471,9 @@ func convertToIndexScan(ds *logicalop.DataSource, prop *property.PhysicalPropert // Case 3: both if (ds.TableInfo.GetPartitionInfo() != nil && !is.Index.Global) || candidate.matchPropResult == property.PropMatchedNeedMergeSort { - byItems := make([]*util.ByItems, 0, len(prop.SortItems)) - for _, si := range prop.SortItems { + sortItems := prop.GetSortItemsForKeepOrder() + byItems := make([]*util.ByItems, 0, len(sortItems)) + for _, si := range sortItems { byItems = append(byItems, &util.ByItems{ Expr: si.Col, Desc: si.Desc, @@ -3265,8 +3350,8 @@ func getOriginalPhysicalIndexScan(ds *logicalop.DataSource, prop *property.Physi if usedStats != nil && usedStats.GetUsedInfo(is.physicalTableID) != nil { is.usedStatsInfo = usedStats.GetUsedInfo(is.physicalTableID) } - if isMatchProp { - is.Desc = prop.SortItems[0].Desc + if prop.NeedKeepOrder() { + is.Desc = prop.GetSortDescForKeepOrder() is.KeepOrder = true } return is diff --git a/pkg/planner/core/hint_test.go b/pkg/planner/core/hint_test.go index 776289a6d0155..126c8dd5ff651 100644 --- a/pkg/planner/core/hint_test.go +++ b/pkg/planner/core/hint_test.go @@ -90,35 +90,30 @@ func TestSetVarPartialOrderedIndexForTopN(t *testing.T) { testKit.MustExec(`use test`) // Test default value - testKit.MustQuery(`select @@tidb_opt_partial_ordered_index_for_topn`).Check(testkit.Rows("0")) + testKit.MustQuery(`select @@tidb_opt_partial_ordered_index_for_topn`).Check(testkit.Rows("DISABLE")) // Test set_var hint changes the value during query execution - testKit.MustExec(`set @@tidb_opt_partial_ordered_index_for_topn = 0`) - testKit.MustQuery(`select /*+ set_var(tidb_opt_partial_ordered_index_for_topn=ON) */ @@tidb_opt_partial_ordered_index_for_topn`).Check(testkit.Rows("1")) + testKit.MustExec(`set @@tidb_opt_partial_ordered_index_for_topn = DISABLE`) + testKit.MustQuery(`select /*+ set_var(tidb_opt_partial_ordered_index_for_topn=COST) */ @@tidb_opt_partial_ordered_index_for_topn`).Check(testkit.Rows("COST")) // Value should be restored after query - testKit.MustQuery(`select @@tidb_opt_partial_ordered_index_for_topn`).Check(testkit.Rows("0")) + testKit.MustQuery(`select @@tidb_opt_partial_ordered_index_for_topn`).Check(testkit.Rows("DISABLE")) // Test set_var hint with OFF - testKit.MustExec(`set @@tidb_opt_partial_ordered_index_for_topn = 1`) - testKit.MustQuery(`select /*+ set_var(tidb_opt_partial_ordered_index_for_topn=OFF) */ @@tidb_opt_partial_ordered_index_for_topn`).Check(testkit.Rows("0")) + testKit.MustExec(`set @@tidb_opt_partial_ordered_index_for_topn = COST`) + testKit.MustQuery(`select /*+ set_var(tidb_opt_partial_ordered_index_for_topn=DISABLE) */ @@tidb_opt_partial_ordered_index_for_topn`).Check(testkit.Rows("DISABLE")) // Value should be restored after query - testKit.MustQuery(`select @@tidb_opt_partial_ordered_index_for_topn`).Check(testkit.Rows("1")) - - // Test set_var hint with numeric values - testKit.MustExec(`set @@tidb_opt_partial_ordered_index_for_topn = 0`) - testKit.MustQuery(`select /*+ set_var(tidb_opt_partial_ordered_index_for_topn=1) */ @@tidb_opt_partial_ordered_index_for_topn`).Check(testkit.Rows("1")) - testKit.MustQuery(`select @@tidb_opt_partial_ordered_index_for_topn`).Check(testkit.Rows("0")) + testKit.MustQuery(`select @@tidb_opt_partial_ordered_index_for_topn`).Check(testkit.Rows("COST")) // Test set_var hint with multiple queries testKit.MustExec(`create table t(a int, b varchar(10), index idx_b(b(5)));`) - testKit.MustExec(`set @@tidb_opt_partial_ordered_index_for_topn = 0`) - testKit.MustExec(`select /*+ set_var(tidb_opt_partial_ordered_index_for_topn=ON) */ * from t order by b limit 10;`) - testKit.MustQuery(`select @@tidb_opt_partial_ordered_index_for_topn`).Check(testkit.Rows("0")) + testKit.MustExec(`set @@tidb_opt_partial_ordered_index_for_topn = DISABLE`) + testKit.MustExec(`select /*+ set_var(tidb_opt_partial_ordered_index_for_topn=COST) */ * from t order by b limit 10;`) + testKit.MustQuery(`select @@tidb_opt_partial_ordered_index_for_topn`).Check(testkit.Rows("DISABLE")) // Test with EXPLAIN (should not change the value) - testKit.MustExec(`set @@tidb_opt_partial_ordered_index_for_topn = 0`) - testKit.MustExec(`explain select /*+ set_var(tidb_opt_partial_ordered_index_for_topn=ON) */ * from t order by b limit 10;`) - testKit.MustQuery(`select @@tidb_opt_partial_ordered_index_for_topn`).Check(testkit.Rows("0")) + testKit.MustExec(`set @@tidb_opt_partial_ordered_index_for_topn = DISABLE`) + testKit.MustExec(`explain select /*+ set_var(tidb_opt_partial_ordered_index_for_topn=COST) */ * from t order by b limit 10;`) + testKit.MustQuery(`select @@tidb_opt_partial_ordered_index_for_topn`).Check(testkit.Rows("DISABLE")) }) } diff --git a/pkg/planner/core/operator/logicalop/logical_projection.go b/pkg/planner/core/operator/logicalop/logical_projection.go index 10d418b6748df..b564fcc0b4abf 100644 --- a/pkg/planner/core/operator/logicalop/logical_projection.go +++ b/pkg/planner/core/operator/logicalop/logical_projection.go @@ -567,18 +567,49 @@ func (p *LogicalProjection) buildSchemaByExprs(selfSchema *expression.Schema) *e // When a sort column will be replaced by a constant, we just remove it. func (p *LogicalProjection) TryToGetChildProp(prop *property.PhysicalProperty) (*property.PhysicalProperty, bool) { newProp := prop.CloneEssentialFields() - newCols := make([]property.SortItem, 0, len(prop.SortItems)) - for _, col := range prop.SortItems { - idx := p.Schema().ColumnIndex(col.Col) + if prop.SortItems != nil { + newSortItems, ok := p.tryTransformSortItems(prop.SortItems) + if !ok { + return nil, false + } + newProp.SortItems = newSortItems + } + if prop.PartialOrderInfo != nil { + newPartialOrderItems, ok := p.tryTransformSortItemPtrs(prop.PartialOrderInfo.SortItems) + if !ok { + return nil, false + } + newProp.PartialOrderInfo = &property.PartialOrderInfo{SortItems: newPartialOrderItems} + } + return newProp, true +} + +func (p *LogicalProjection) tryTransformSortItems(items []property.SortItem) ([]property.SortItem, bool) { + newItems := make([]property.SortItem, 0, len(items)) + for _, item := range items { + idx := p.Schema().ColumnIndex(item.Col) switch expr := p.Exprs[idx].(type) { case *expression.Column: - newCols = append(newCols, property.SortItem{Col: expr, Desc: col.Desc}) + newItems = append(newItems, property.SortItem{Col: expr, Desc: item.Desc}) case *expression.ScalarFunction: return nil, false } } - newProp.SortItems = newCols - return newProp, true + return newItems, true +} + +func (p *LogicalProjection) tryTransformSortItemPtrs(items []*property.SortItem) ([]*property.SortItem, bool) { + newItems := make([]*property.SortItem, 0, len(items)) + for _, item := range items { + idx := p.Schema().ColumnIndex(item.Col) + switch expr := p.Exprs[idx].(type) { + case *expression.Column: + newItems = append(newItems, &property.SortItem{Col: expr, Desc: item.Desc}) + case *expression.ScalarFunction: + return nil, false + } + } + return newItems, true } func (p *LogicalProjection) getGroupNDVs(colGroups [][]*expression.Column, childProfile *property.StatsInfo, selfSchema *expression.Schema) []property.GroupNDV { diff --git a/pkg/planner/core/physical_plans.go b/pkg/planner/core/physical_plans.go index 5c96a984e4fea..b839f4c7382b7 100644 --- a/pkg/planner/core/physical_plans.go +++ b/pkg/planner/core/physical_plans.go @@ -1280,6 +1280,11 @@ type PhysicalTopN struct { PartitionBy []property.SortItem Offset uint64 Count uint64 + + // PrefixCol is the prefix index column for partial order optimization. + PrefixCol *expression.Column + // PrefixLen is the prefix index length in bytes. + PrefixLen int } // GetPartitionBy returns partition by fields @@ -1323,7 +1328,8 @@ func (lt *PhysicalTopN) MemoryUsage() (sum int64) { return } - sum = lt.BasePhysicalPlan.MemoryUsage() + size.SizeOfSlice + int64(cap(lt.ByItems))*size.SizeOfPointer + size.SizeOfUint64*2 + sum = lt.BasePhysicalPlan.MemoryUsage() + size.SizeOfSlice + int64(cap(lt.ByItems))*size.SizeOfPointer + + size.SizeOfUint64*2 + size.SizeOfInt64 + size.SizeOfInt for _, byItem := range lt.ByItems { sum += byItem.MemoryUsage() } @@ -2074,6 +2080,11 @@ type PhysicalLimit struct { PartitionBy []property.SortItem Offset uint64 Count uint64 + + // PrefixCol is the prefix index column for partial order optimization. + PrefixCol *expression.Column + // PrefixLen is the prefix index length in bytes. + PrefixLen int } // GetPartitionBy returns partition by fields @@ -2104,7 +2115,7 @@ func (p *PhysicalLimit) MemoryUsage() (sum int64) { return } - sum = p.physicalSchemaProducer.MemoryUsage() + size.SizeOfUint64*2 + sum = p.physicalSchemaProducer.MemoryUsage() + size.SizeOfUint64*2 + size.SizeOfInt64 + size.SizeOfInt return } diff --git a/pkg/planner/core/plan_clone_generated.go b/pkg/planner/core/plan_clone_generated.go index 00bf501d8483a..f633eb61409cd 100644 --- a/pkg/planner/core/plan_clone_generated.go +++ b/pkg/planner/core/plan_clone_generated.go @@ -144,6 +144,9 @@ func (op *PhysicalTopN) CloneForPlanCache(newCtx base.PlanContext) (base.Plan, b cloned.BasePhysicalPlan = *basePlan cloned.ByItems = util.CloneByItemss(op.ByItems) cloned.PartitionBy = util.CloneSortItems(op.PartitionBy) + if op.PrefixCol != nil { + cloned.PrefixCol = op.PrefixCol.Clone().(*expression.Column) + } return cloned, true } @@ -323,6 +326,9 @@ func (op *PhysicalLimit) CloneForPlanCache(newCtx base.PlanContext) (base.Plan, } cloned.physicalSchemaProducer = *basePlan cloned.PartitionBy = util.CloneSortItems(op.PartitionBy) + if op.PrefixCol != nil { + cloned.PrefixCol = op.PrefixCol.Clone().(*expression.Column) + } return cloned, true } diff --git a/pkg/planner/core/plan_to_pb.go b/pkg/planner/core/plan_to_pb.go index d974349fb7594..04245609de682 100644 --- a/pkg/planner/core/plan_to_pb.go +++ b/pkg/planner/core/plan_to_pb.go @@ -243,6 +243,14 @@ func (p *PhysicalLimit) ToPB(ctx *base.BuildPBContext, storeType kv.StoreType) ( for _, item := range p.PartitionBy { limitExec.PartitionBy = append(limitExec.PartitionBy, expression.SortByItemToPB(ctx.GetExprCtx().GetEvalCtx(), client, item.Col.Clone(), item.Desc)) } + if p.PrefixCol != nil { + truncateKeyExprs := []expression.Expression{p.PrefixCol} + truncateKeyExprsPB, err := expression.ExpressionsToPBList(ctx.GetExprCtx().GetEvalCtx(), truncateKeyExprs, client) + if err != nil { + return nil, err + } + limitExec.TruncateKeyExpr = truncateKeyExprsPB + } if storeType == kv.TiFlash { var err error limitExec.Child, err = p.Children()[0].ToPB(ctx, storeType) diff --git a/pkg/planner/core/task.go b/pkg/planner/core/task.go index 53c249223f409..1b1b6a258ad2c 100644 --- a/pkg/planner/core/task.go +++ b/pkg/planner/core/task.go @@ -1043,6 +1043,11 @@ func (p *PhysicalTopN) pushLimitDownToTiDBCop(copTsk *CopTask) (base.Task, bool) // Attach2Task implements the PhysicalPlan interface. func (p *PhysicalTopN) Attach2Task(tasks ...base.Task) base.Task { t := tasks[0].Copy() + if copTask, ok := t.(*CopTask); ok { + if copTask.partialOrderMatchResult != nil && copTask.partialOrderMatchResult.Matched { + return handlePartialOrderTopN(p, copTask) + } + } cols := make([]*expression.Column, 0, len(p.ByItems)) for _, item := range p.ByItems { cols = append(cols, expression.ExtractColumns(item.Expr)...) @@ -1080,6 +1085,53 @@ func (p *PhysicalTopN) Attach2Task(tasks ...base.Task) base.Task { return attachPlan2Task(p, rootTask) } +// handlePartialOrderTopN handles the partial order TopN scenario. +// +// Case 1: two-phase TopN, where TiDB keeps TopN and TiKV applies a partial-order Limit: +// +// TopN(with partial info) +// └─IndexLookUp +// └─Limit(with partial info) +// +// Case 2: one-phase TopN, where the whole TopN can be executed in the coprocessor: +// +// TopN(with partial info) +// ├─IndexPlan +// └─TablePlan +func handlePartialOrderTopN(p *PhysicalTopN, copTask *CopTask) base.Task { + matchResult := copTask.partialOrderMatchResult + partialOrderedLimit := p.Count + p.Offset + p.PrefixCol = matchResult.PrefixCol + p.PrefixLen = matchResult.PrefixLen + + canPushLimit := len(copTask.idxMergePartPlans) == 0 && + !copTask.indexPlanFinished && + len(copTask.rootTaskConds) == 0 && + copTask.indexPlan != nil + if canPushLimit { + maxX := estimateMaxXForPartialOrder(p.SCtx(), copTask) + estimatedRows := float64(partialOrderedLimit) + float64(maxX) + childProfile := copTask.indexPlan.StatsInfo() + limitStats := util.DeriveLimitStats(childProfile, estimatedRows) + + pushedDownLimit := PhysicalLimit{ + Count: partialOrderedLimit, + PrefixCol: matchResult.PrefixCol, + PrefixLen: matchResult.PrefixLen, + }.Init(p.SCtx(), limitStats, p.QueryBlockOffset()) + pushedDownLimit.SetChildren(copTask.indexPlan) + pushedDownLimit.SetSchema(copTask.indexPlan.Schema()) + copTask.indexPlan = pushedDownLimit + } + rootTask := copTask.ConvertToRootTask(p.SCtx()) + return attachPlan2Task(p, rootTask) +} + +// estimateMaxXForPartialOrder estimates the extra rows X to read for partial order optimization. +func estimateMaxXForPartialOrder(_ base.PlanContext, _ *CopTask) uint64 { + return 0 +} + // Attach2Task implements the PhysicalPlan interface. func (p *PhysicalExpand) Attach2Task(tasks ...base.Task) base.Task { t := tasks[0].Copy() diff --git a/pkg/planner/core/task_base.go b/pkg/planner/core/task_base.go index fd00eb0b4eba4..86c53421e74fb 100644 --- a/pkg/planner/core/task_base.go +++ b/pkg/planner/core/task_base.go @@ -260,6 +260,9 @@ type CopTask struct { // expectCnt is the expected row count of upper task, 0 for unlimited. // It's used for deciding whether using paging distsql. expectCnt uint64 + + // partialOrderMatchResult stores the match result for partial order optimization. + partialOrderMatchResult *property.PartialOrderMatchResult } // Invalid implements Task interface. diff --git a/pkg/planner/property/physical_property.go b/pkg/planner/property/physical_property.go index 7b95959d71dfe..2d48ce3757b8c 100644 --- a/pkg/planner/property/physical_property.go +++ b/pkg/planner/property/physical_property.go @@ -287,6 +287,42 @@ type PhysicalProperty struct { *expression.VectorHelper TopK uint32 } + + // PartialOrderInfo is used for TopN's partial order optimization. + // When this field is not nil, it indicates that prefix index can be used + // to provide partial order for TopN. + PartialOrderInfo *PartialOrderInfo +} + +// PartialOrderInfo records information needed for partial order optimization. +// When PhysicalProperty.PartialOrderInfo is not nil, it indicates that +// prefix index can be used to provide partial order. +type PartialOrderInfo struct { + // SortItems are the ORDER BY columns from TopN. + SortItems []*SortItem +} + +// AllSameOrder checks if all the items have same order. +func (p *PartialOrderInfo) AllSameOrder() (isSame bool, desc bool) { + if len(p.SortItems) == 0 { + return true, false + } + for i := 1; i < len(p.SortItems); i++ { + if p.SortItems[i].Desc != p.SortItems[i-1].Desc { + return + } + } + return true, p.SortItems[0].Desc +} + +// PartialOrderMatchResult records the result of matching partial order property with an access path. +type PartialOrderMatchResult struct { + // Matched indicates whether this path can provide partial order. + Matched bool + // PrefixCol is the last and only one prefix column in the matched index. + PrefixCol *expression.Column + // PrefixLen is the prefix length in bytes for prefix index. + PrefixLen int } // NewPhysicalProperty builds property from columns. @@ -388,12 +424,44 @@ func (p *PhysicalProperty) IsSortItemEmpty() bool { return len(p.SortItems) == 0 } +// NeedKeepOrder returns whether the property requires maintaining order. +func (p *PhysicalProperty) NeedKeepOrder() bool { + return !p.IsSortItemEmpty() || p.PartialOrderInfo != nil +} + +// GetSortDescForKeepOrder returns the sort direction for keep-order scans. +func (p *PhysicalProperty) GetSortDescForKeepOrder() bool { + if p.PartialOrderInfo != nil && len(p.PartialOrderInfo.SortItems) > 0 { + _, desc := p.PartialOrderInfo.AllSameOrder() + return desc + } + _, desc := p.AllSameOrder() + return desc +} + +// GetSortItemsForKeepOrder returns the sort items used for keep-order scans. +func (p *PhysicalProperty) GetSortItemsForKeepOrder() []SortItem { + if p.PartialOrderInfo != nil && len(p.PartialOrderInfo.SortItems) > 0 { + items := make([]SortItem, 0, len(p.PartialOrderInfo.SortItems)) + for _, si := range p.PartialOrderInfo.SortItems { + items = append(items, *si) + } + return items + } + return p.SortItems +} + // HashCode calculates hash code for a PhysicalProperty object. func (p *PhysicalProperty) HashCode() []byte { if p.hashcode != nil { return p.hashcode } hashcodeSize := 8 + 8 + 8 + (16+8)*len(p.SortItems) + 8 + if p.PartialOrderInfo != nil { + hashcodeSize += (16 + 8) * len(p.PartialOrderInfo.SortItems) + } else { + hashcodeSize += 8 + } p.hashcode = make([]byte, 0, hashcodeSize) if p.CanAddEnforcer { p.hashcode = codec.EncodeInt(p.hashcode, 1) @@ -423,6 +491,19 @@ func (p *PhysicalProperty) HashCode() []byte { } } p.hashcode = append(p.hashcode, codec.EncodeInt(nil, int64(p.CTEProducerStatus))...) + if p.PartialOrderInfo != nil { + p.hashcode = codec.EncodeInt(p.hashcode, 1) + for _, item := range p.PartialOrderInfo.SortItems { + p.hashcode = append(p.hashcode, item.Col.HashCode()...) + if item.Desc { + p.hashcode = codec.EncodeInt(p.hashcode, 1) + } else { + p.hashcode = codec.EncodeInt(p.hashcode, 0) + } + } + } else { + p.hashcode = codec.EncodeInt(p.hashcode, 0) + } return p.hashcode } @@ -443,6 +524,7 @@ func (p *PhysicalProperty) CloneEssentialFields() *PhysicalProperty { MPPPartitionCols: p.MPPPartitionCols, RejectSort: p.RejectSort, CTEProducerStatus: p.CTEProducerStatus, + PartialOrderInfo: p.PartialOrderInfo, } return prop } diff --git a/pkg/planner/util/path.go b/pkg/planner/util/path.go index 671612c35fafd..55d227e3acf4d 100644 --- a/pkg/planner/util/path.go +++ b/pkg/planner/util/path.go @@ -101,6 +101,8 @@ type AccessPath struct { Forced bool ForceKeepOrder bool ForceNoKeepOrder bool + // ForcePartialOrder means whether to force using current path with partial order optimization. + ForcePartialOrder bool // IsSingleScan indicates whether the path is a single index/table scan or table access after index scan. IsSingleScan bool @@ -152,6 +154,7 @@ func (path *AccessPath) Clone() *AccessPath { Forced: path.Forced, ForceKeepOrder: path.ForceKeepOrder, ForceNoKeepOrder: path.ForceNoKeepOrder, + ForcePartialOrder: path.ForcePartialOrder, IsSingleScan: path.IsSingleScan, IsUkShardIndexPath: path.IsUkShardIndexPath, KeepIndexMergeORSourceFilter: path.KeepIndexMergeORSourceFilter, diff --git a/pkg/sessionctx/variable/session.go b/pkg/sessionctx/variable/session.go index 3a481465926ec..b19e250873778 100644 --- a/pkg/sessionctx/variable/session.go +++ b/pkg/sessionctx/variable/session.go @@ -1607,7 +1607,8 @@ type SessionVars struct { // When set to true, `col is (not) null`(`col` is index prefix column) is regarded as index filter rather than table filter. OptPrefixIndexSingleScan bool // OptPartialOrderedIndexForTopN indicates whether to enable partial ordered index optimization for TOPN queries. - OptPartialOrderedIndexForTopN bool + // Valid values: "DISABLE" and "COST". + OptPartialOrderedIndexForTopN string // chunkPool Several chunks and columns are cached chunkPool chunk.Allocator @@ -1954,6 +1955,11 @@ func (s *SessionVars) RaiseWarningWhenMPPEnforced(warning string) { } } +// IsPartialOrderedIndexForTopNEnabled indicates whether partial ordered index optimization for TopN is enabled. +func (s *SessionVars) IsPartialOrderedIndexForTopNEnabled() bool { + return s.OptPartialOrderedIndexForTopN == "COST" +} + // CheckAndGetTxnScope will return the transaction scope we should use in the current session. func (s *SessionVars) CheckAndGetTxnScope() string { if s.InRestrictedSQL || !EnableLocalTxn.Load() { diff --git a/pkg/sessionctx/variable/session_test.go b/pkg/sessionctx/variable/session_test.go index 2224d88cde6ee..03a898ab41a8c 100644 --- a/pkg/sessionctx/variable/session_test.go +++ b/pkg/sessionctx/variable/session_test.go @@ -741,48 +741,48 @@ func TestTiDBOptPartialOrderedIndexForTopNSessionAndGlobal(t *testing.T) { tk.MustExec("use test") // Test default value - tk.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("0")) - tk.MustQuery("select @@global.tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("0")) + tk.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("DISABLE")) + tk.MustQuery("select @@global.tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("DISABLE")) // Test session scope - tk.MustExec("set @@tidb_opt_partial_ordered_index_for_topn = ON") - tk.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("1")) - tk.MustQuery("select @@session.tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("1")) + tk.MustExec("set @@tidb_opt_partial_ordered_index_for_topn = COST") + tk.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("COST")) + tk.MustQuery("select @@session.tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("COST")) // Global should not be affected - tk.MustQuery("select @@global.tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("0")) + tk.MustQuery("select @@global.tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("DISABLE")) - tk.MustExec("set @@tidb_opt_partial_ordered_index_for_topn = OFF") - tk.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("0")) + tk.MustExec("set @@tidb_opt_partial_ordered_index_for_topn = DISABLE") + tk.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("DISABLE")) // Test global scope - tk.MustExec("set @@global.tidb_opt_partial_ordered_index_for_topn = ON") - tk.MustQuery("select @@global.tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("1")) + tk.MustExec("set @@global.tidb_opt_partial_ordered_index_for_topn = COST") + tk.MustQuery("select @@global.tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("COST")) // New session should inherit global value tk1 := testkit.NewTestKit(t, store) tk1.MustExec("use test") - tk1.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("1")) + tk1.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("COST")) // Session value should override global value - tk.MustExec("set @@tidb_opt_partial_ordered_index_for_topn = OFF") - tk.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("0")) - // Global should still be ON - tk.MustQuery("select @@global.tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("1")) - - // Test different value formats (only 0, 1, ON, OFF are allowed) - tk.MustExec("set @@tidb_opt_partial_ordered_index_for_topn = 1") - tk.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("1")) - tk.MustExec("set @@tidb_opt_partial_ordered_index_for_topn = 0") - tk.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("0")) - tk.MustExec("set @@tidb_opt_partial_ordered_index_for_topn = 'ON'") - tk.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("1")) - tk.MustExec("set @@tidb_opt_partial_ordered_index_for_topn = 'OFF'") - tk.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("0")) - tk.MustExec("set @@tidb_opt_partial_ordered_index_for_topn = 'on'") - tk.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("1")) - tk.MustExec("set @@tidb_opt_partial_ordered_index_for_topn = 'off'") - tk.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("0")) + tk.MustExec("set @@tidb_opt_partial_ordered_index_for_topn = DISABLE") + tk.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("DISABLE")) + // Global should still be COST + tk.MustQuery("select @@global.tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("COST")) + + // Test different value formats (only DISABLE and COST are allowed) + tk.MustExec("set @@tidb_opt_partial_ordered_index_for_topn = 'COST'") + tk.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("COST")) + tk.MustExec("set @@tidb_opt_partial_ordered_index_for_topn = 'DISABLE'") + tk.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("DISABLE")) + tk.MustExec("set @@tidb_opt_partial_ordered_index_for_topn = 'cost'") + tk.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("COST")) + tk.MustExec("set @@tidb_opt_partial_ordered_index_for_topn = 'disable'") + tk.MustQuery("select @@tidb_opt_partial_ordered_index_for_topn").Check(testkit.Rows("DISABLE")) // Test disallowed values + require.Error(t, tk.ExecToErr("set @@tidb_opt_partial_ordered_index_for_topn = 'ON'")) + require.Error(t, tk.ExecToErr("set @@tidb_opt_partial_ordered_index_for_topn = 'OFF'")) + require.Error(t, tk.ExecToErr("set @@tidb_opt_partial_ordered_index_for_topn = 1")) + require.Error(t, tk.ExecToErr("set @@tidb_opt_partial_ordered_index_for_topn = 0")) require.Error(t, tk.ExecToErr("set @@tidb_opt_partial_ordered_index_for_topn = 'true'")) require.Error(t, tk.ExecToErr("set @@tidb_opt_partial_ordered_index_for_topn = 'false'")) require.Error(t, tk.ExecToErr("set @@tidb_opt_partial_ordered_index_for_topn = 2")) @@ -792,9 +792,11 @@ func TestTiDBOptPartialOrderedIndexForTopNSessionAndGlobal(t *testing.T) { // Verify the field is accessible in SessionVars vars := tk.Session().GetSessionVars() - require.False(t, vars.OptPartialOrderedIndexForTopN) - tk.MustExec("set @@tidb_opt_partial_ordered_index_for_topn = ON") - require.True(t, vars.OptPartialOrderedIndexForTopN) + require.Equal(t, "DISABLE", vars.OptPartialOrderedIndexForTopN) + require.False(t, vars.IsPartialOrderedIndexForTopNEnabled()) + tk.MustExec("set @@tidb_opt_partial_ordered_index_for_topn = COST") + require.Equal(t, "COST", vars.OptPartialOrderedIndexForTopN) + require.True(t, vars.IsPartialOrderedIndexForTopNEnabled()) } func TestPerformanceSchemaSessionConnectAttrsSizeGlobalSQL(t *testing.T) { diff --git a/pkg/sessionctx/variable/sysvar.go b/pkg/sessionctx/variable/sysvar.go index 299f9891f2f96..03b12881c1b4f 100644 --- a/pkg/sessionctx/variable/sysvar.go +++ b/pkg/sessionctx/variable/sysvar.go @@ -2901,16 +2901,15 @@ var defaultSysVars = []*SysVar{ s.OptPrefixIndexSingleScan = TiDBOptOn(val) return nil }}, - {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptPartialOrderedIndexForTopN, Value: BoolToOnOff(DefTiDBOptPartialOrderedIndexForTopN), Type: TypeBool, IsHintUpdatableVerified: true, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptPartialOrderedIndexForTopN, Value: DefTiDBOptPartialOrderedIndexForTopN, Type: TypeEnum, PossibleValues: []string{"DISABLE", "COST"}, IsHintUpdatableVerified: true, Validation: func(_ *SessionVars, normalizedValue string, originalValue string, _ ScopeFlag) (string, error) { - // Only allow exact values: 0, 1, ON, OFF (case-insensitive). lowerValue := strings.ToLower(strings.TrimSpace(originalValue)) - if lowerValue != "0" && lowerValue != "1" && lowerValue != "on" && lowerValue != "off" { + if lowerValue != "disable" && lowerValue != "cost" { return normalizedValue, ErrWrongValueForVar.GenWithStackByArgs(TiDBOptPartialOrderedIndexForTopN, originalValue) } return normalizedValue, nil }, SetSession: func(s *SessionVars, val string) error { - s.OptPartialOrderedIndexForTopN = TiDBOptOn(val) + s.OptPartialOrderedIndexForTopN = strings.ToUpper(val) return nil }}, {Scope: ScopeGlobal, Name: TiDBExternalTS, Value: strconv.FormatInt(DefTiDBExternalTS, 10), SetGlobal: func(ctx context.Context, s *SessionVars, val string) error { diff --git a/pkg/sessionctx/variable/sysvar_test.go b/pkg/sessionctx/variable/sysvar_test.go index 5d3ece5484f07..1ac5e25d018de 100644 --- a/pkg/sessionctx/variable/sysvar_test.go +++ b/pkg/sessionctx/variable/sysvar_test.go @@ -1866,35 +1866,43 @@ func TestTiDBOptPartialOrderedIndexForTopN(t *testing.T) { require.True(t, sv.HasSessionScope()) require.True(t, sv.HasGlobalScope()) require.True(t, sv.IsHintUpdatableVerified) - require.Equal(t, TypeBool, sv.Type) - require.Equal(t, "OFF", sv.Value) + require.Equal(t, TypeEnum, sv.Type) + require.Equal(t, "DISABLE", sv.Value) vars := NewSessionVars(nil) vars.GlobalVarsAccessor = NewMockGlobalAccessor4Tests() - val, err := sv.Validate(vars, "ON", ScopeSession) + val, err := sv.Validate(vars, "COST", ScopeSession) require.NoError(t, err) - require.Equal(t, "ON", val) + require.Equal(t, "COST", val) - val, err = sv.Validate(vars, "on", ScopeSession) + val, err = sv.Validate(vars, "cost", ScopeSession) require.NoError(t, err) - require.Equal(t, "ON", val) + require.Equal(t, "COST", val) - val, err = sv.Validate(vars, "OFF", ScopeSession) + val, err = sv.Validate(vars, "DISABLE", ScopeSession) require.NoError(t, err) - require.Equal(t, "OFF", val) + require.Equal(t, "DISABLE", val) - val, err = sv.Validate(vars, "off", ScopeSession) + val, err = sv.Validate(vars, "disable", ScopeSession) require.NoError(t, err) - require.Equal(t, "OFF", val) + require.Equal(t, "DISABLE", val) - val, err = sv.Validate(vars, "1", ScopeSession) - require.NoError(t, err) - require.Equal(t, "ON", val) + _, err = sv.Validate(vars, "ON", ScopeSession) + require.Error(t, err) + require.Contains(t, err.Error(), "can't be set to the value of") - val, err = sv.Validate(vars, "0", ScopeSession) - require.NoError(t, err) - require.Equal(t, "OFF", val) + _, err = sv.Validate(vars, "OFF", ScopeSession) + require.Error(t, err) + require.Contains(t, err.Error(), "can't be set to the value of") + + _, err = sv.Validate(vars, "1", ScopeSession) + require.Error(t, err) + require.Contains(t, err.Error(), "can't be set to the value of") + + _, err = sv.Validate(vars, "0", ScopeSession) + require.Error(t, err) + require.Contains(t, err.Error(), "can't be set to the value of") _, err = sv.Validate(vars, "true", ScopeSession) require.Error(t, err) @@ -1920,11 +1928,13 @@ func TestTiDBOptPartialOrderedIndexForTopN(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "can't be set to the value of") - err = sv.SetSessionFromHook(vars, "ON") + err = sv.SetSessionFromHook(vars, "COST") require.NoError(t, err) - require.True(t, vars.OptPartialOrderedIndexForTopN) + require.Equal(t, "COST", vars.OptPartialOrderedIndexForTopN) + require.True(t, vars.IsPartialOrderedIndexForTopNEnabled()) - err = sv.SetSessionFromHook(vars, "OFF") + err = sv.SetSessionFromHook(vars, "DISABLE") require.NoError(t, err) - require.False(t, vars.OptPartialOrderedIndexForTopN) + require.Equal(t, "DISABLE", vars.OptPartialOrderedIndexForTopN) + require.False(t, vars.IsPartialOrderedIndexForTopNEnabled()) } diff --git a/pkg/sessionctx/variable/tidb_vars.go b/pkg/sessionctx/variable/tidb_vars.go index 377f419138c62..4e1c1e2b7e9e9 100644 --- a/pkg/sessionctx/variable/tidb_vars.go +++ b/pkg/sessionctx/variable/tidb_vars.go @@ -1595,7 +1595,7 @@ const ( DefTiDBGOGCMaxValue = 500 DefTiDBGOGCMinValue = 100 DefTiDBOptPrefixIndexSingleScan = true - DefTiDBOptPartialOrderedIndexForTopN = false + DefTiDBOptPartialOrderedIndexForTopN = "DISABLE" DefTiDBEnableAsyncMergeGlobalStats = true DefTiDBExternalTS = 0 DefTiDBEnableExternalTSRead = false diff --git a/tests/integrationtest/r/planner/core/partial_order_topn.result b/tests/integrationtest/r/planner/core/partial_order_topn.result new file mode 100644 index 0000000000000..2bff94f315b87 --- /dev/null +++ b/tests/integrationtest/r/planner/core/partial_order_topn.result @@ -0,0 +1,612 @@ +drop table if exists t_varchar, t_char, t_text, t_multi_col, t_no_prefix; +create table t_varchar ( +id int primary key auto_increment, +name varchar(255), +data varchar(100), +index idx_name_prefix (name(10)) +); +create table t_char ( +id int primary key auto_increment, +code char(50), +value int, +index idx_code_prefix (code(8)) +); +create table t_text ( +id int primary key auto_increment, +content text, +score int, +index idx_content_prefix (content(20)) +); +create table t_multi_col ( +id int primary key auto_increment, +a int, +b varchar(100), +c varchar(100), +index idx_a_b_prefix (a, b(15)), +index idx_a_b_c_prefix (a, b, c(10)) +); +create table t_no_prefix ( +id int primary key auto_increment, +name varchar(255), +data varchar(100), +index idx_name (name) +); +insert into t_varchar (name, data) values +('apple', 'fruit'), +('apricot', 'fruit'), +('application', 'software'), +('banana', 'fruit'), +('blueberry', 'fruit'), +('cherry', 'fruit'), +('date', 'fruit'), +('elderberry', 'fruit'), +('fig', 'fruit'), +('grape', 'fruit'); +insert into t_char (code, value) values +('ABC12345', 100), +('ABC12346', 200), +('ABC12347', 300), +('DEF12345', 400), +('DEF12346', 500), +('GHI12345', 600), +('GHI12346', 700), +('JKL12345', 800), +('JKL12346', 900), +('MNO12345', 1000); +insert into t_text (content, score) values +('This is a long text content for testing', 10), +('Another long text content here', 20), +('Some more text data for testing purposes', 30), +('Text content with different prefix', 40), +('Testing partial order optimization', 50), +('More test data with various lengths', 60), +('Short text', 70), +('Medium length text content', 80), +('Very long text content that exceeds the prefix length significantly', 90), +('Final test text content', 100); +insert into t_multi_col (a, b, c) values +(1, 'alpha', 'first'), +(1, 'beta', 'second'), +(1, 'gamma', 'third'), +(2, 'alpha', 'fourth'), +(2, 'beta', 'fifth'), +(2, 'gamma', 'sixth'), +(3, 'alpha', 'seventh'), +(3, 'beta', 'eighth'), +(3, 'gamma', 'ninth'), +(3, 'delta', 'tenth'); +insert into t_no_prefix (name, data) values +('apple', 'fruit'), +('apricot', 'fruit'), +('application', 'software'), +('banana', 'fruit'), +('blueberry', 'fruit'); +select @@tidb_opt_partial_ordered_index_for_topn; +@@tidb_opt_partial_ordered_index_for_topn +DISABLE +set @@tidb_opt_partial_ordered_index_for_topn = 'DISABLE'; +explain format='brief' select * from t_varchar order by name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5 +└─TableReader 5.00 root data:TopN + └─TopN 5.00 cop[tikv] planner__core__partial_order_topn.t_varchar.name, offset:0, count:5 + └─TableFullScan 10000.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select * from t_varchar where data = 'fruit' order by name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5 +└─TableReader 5.00 root data:TopN + └─TopN 5.00 cop[tikv] planner__core__partial_order_topn.t_varchar.name, offset:0, count:5 + └─Selection 10.00 cop[tikv] eq(planner__core__partial_order_topn.t_varchar.data, "fruit") + └─TableFullScan 10000.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select * from t_multi_col where a = 1 order by b limit 5; +id estRows task access object operator info +IndexLookUp 5.00 root limit embedded(offset:0, count:5) +├─Limit(Build) 5.00 cop[tikv] offset:0, count:5 +│ └─IndexRangeScan 5.00 cop[tikv] table:t_multi_col, index:idx_a_b_c_prefix(a, b, c) range:[1,1], keep order:true, stats:pseudo +└─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_multi_col keep order:false, stats:pseudo +set @@tidb_opt_partial_ordered_index_for_topn = 'COST'; +select @@tidb_opt_partial_ordered_index_for_topn; +@@tidb_opt_partial_ordered_index_for_topn +COST +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 +└─IndexLookUp 5.00 root + ├─Limit(Build) 5.00 cop[tikv] offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_char, idx_code_prefix) */ * from t_char order by code limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_char.code, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_char.code, prefix_len:8 +└─IndexLookUp 5.00 root + ├─Limit(Build) 5.00 cop[tikv] offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_char.code, prefix_len:8 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_char, index:idx_code_prefix(code) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_char keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_text, idx_content_prefix) */ * from t_text order by content limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_text.content, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_text.content, prefix_len:20 +└─IndexLookUp 5.00 root + ├─Limit(Build) 5.00 cop[tikv] offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_text.content, prefix_len:20 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_text, index:idx_content_prefix(content) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_text keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ * from t_varchar where id=1 order by name limit 5; +id estRows task access object operator info +TopN 1.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 +└─IndexLookUp 1.00 root + ├─Limit(Build) 1.00 cop[tikv] offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 + │ └─Selection 1.00 cop[tikv] eq(planner__core__partial_order_topn.t_varchar.id, 1) + │ └─IndexFullScan 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ * from t_varchar where left(name, 2) = 'ap' order by name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 +└─Selection 8000.00 root eq(left(planner__core__partial_order_topn.t_varchar.name, 2), "ap") + └─IndexLookUp 10000.00 root + ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 10000.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ * from t_varchar where data = 'fruit' order by name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 +└─IndexLookUp 10.00 root + ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:true, stats:pseudo + └─Selection(Probe) 10.00 cop[tikv] eq(planner__core__partial_order_topn.t_varchar.data, "fruit") + └─TableRowIDScan 10000.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_char, idx_code_prefix) */ * from t_char where value > 300 order by code limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_char.code, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_char.code, prefix_len:8 +└─IndexLookUp 3333.33 root + ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t_char, index:idx_code_prefix(code) keep order:true, stats:pseudo + └─Selection(Probe) 3333.33 cop[tikv] gt(planner__core__partial_order_topn.t_char.value, 300) + └─TableRowIDScan 10000.00 cop[tikv] table:t_char keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_text, idx_content_prefix) */ * from t_text where score < 80 order by content limit 3; +id estRows task access object operator info +TopN 3.00 root planner__core__partial_order_topn.t_text.content, offset:0, count:3, prefix_col:planner__core__partial_order_topn.t_text.content, prefix_len:20 +└─IndexLookUp 3323.33 root + ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t_text, index:idx_content_prefix(content) keep order:true, stats:pseudo + └─Selection(Probe) 3323.33 cop[tikv] lt(planner__core__partial_order_topn.t_text.score, 80) + └─TableRowIDScan 10000.00 cop[tikv] table:t_text keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ id, inner_name from (select id, name as inner_name from t_varchar) tmp order by inner_name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 +└─IndexLookUp 5.00 root + ├─Limit(Build) 5.00 cop[tikv] offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_char, idx_code_prefix) */ id, code as c from t_char order by code limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_char.code, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_char.code, prefix_len:8 +└─IndexLookUp 5.00 root + ├─Limit(Build) 5.00 cop[tikv] offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_char.code, prefix_len:8 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_char, index:idx_code_prefix(code) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_char keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ id, name from t_varchar where data = 'fruit' order by name limit 5; +id estRows task access object operator info +Projection 5.00 root planner__core__partial_order_topn.t_varchar.id, planner__core__partial_order_topn.t_varchar.name +└─TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 + └─IndexLookUp 10.00 root + ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:true, stats:pseudo + └─Selection(Probe) 10.00 cop[tikv] eq(planner__core__partial_order_topn.t_varchar.data, "fruit") + └─TableRowIDScan 10000.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_char, idx_code_prefix) */ id, code as c from t_char where value > 200 order by code limit 5; +id estRows task access object operator info +Projection 5.00 root planner__core__partial_order_topn.t_char.id, planner__core__partial_order_topn.t_char.code +└─TopN 5.00 root planner__core__partial_order_topn.t_char.code, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_char.code, prefix_len:8 + └─IndexLookUp 3333.33 root + ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t_char, index:idx_code_prefix(code) keep order:true, stats:pseudo + └─Selection(Probe) 3333.33 cop[tikv] gt(planner__core__partial_order_topn.t_char.value, 200) + └─TableRowIDScan 10000.00 cop[tikv] table:t_char keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ id, name from t_varchar where data = 'fruit' order by name limit 5; +id estRows task access object operator info +Projection 5.00 root planner__core__partial_order_topn.t_varchar.id, planner__core__partial_order_topn.t_varchar.name +└─TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 + └─IndexLookUp 10.00 root + ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:true, stats:pseudo + └─Selection(Probe) 10.00 cop[tikv] eq(planner__core__partial_order_topn.t_varchar.data, "fruit") + └─TableRowIDScan 10000.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_multi_col, idx_a_b_prefix) */ * from t_multi_col where a = 1 order by b limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_multi_col.b, offset:0, count:5 +└─IndexLookUp 5.00 root + ├─IndexRangeScan(Build) 10.00 cop[tikv] table:t_multi_col, index:idx_a_b_prefix(a, b) range:[1,1], keep order:false, stats:pseudo + └─TopN(Probe) 5.00 cop[tikv] planner__core__partial_order_topn.t_multi_col.b, offset:0, count:5 + └─TableRowIDScan 10.00 cop[tikv] table:t_multi_col keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_multi_col, idx_a_b_prefix) */ * from t_multi_col order by a, b limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_multi_col.a, planner__core__partial_order_topn.t_multi_col.b, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_multi_col.b, prefix_len:15 +└─IndexLookUp 5.00 root + ├─Limit(Build) 5.00 cop[tikv] offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_multi_col.b, prefix_len:15 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_multi_col, index:idx_a_b_prefix(a, b) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_multi_col keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_multi_col, idx_a_b_c_prefix) */ * from t_multi_col order by a, b, c limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_multi_col.a, planner__core__partial_order_topn.t_multi_col.b, planner__core__partial_order_topn.t_multi_col.c, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_multi_col.c, prefix_len:10 +└─IndexLookUp 5.00 root + ├─Limit(Build) 5.00 cop[tikv] offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_multi_col.c, prefix_len:10 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_multi_col, index:idx_a_b_c_prefix(a, b, c) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_multi_col keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5 offset 3; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:3, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 +└─IndexLookUp 8.00 root + ├─Limit(Build) 8.00 cop[tikv] offset:0, count:8, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 8.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 10 offset 5; +id estRows task access object operator info +TopN 10.00 root planner__core__partial_order_topn.t_varchar.name, offset:5, count:10, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 +└─IndexLookUp 15.00 root + ├─Limit(Build) 15.00 cop[tikv] offset:0, count:15, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 15.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name desc limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_varchar.name:desc, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 +└─IndexLookUp 5.00 root + ├─Limit(Build) 5.00 cop[tikv] offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:true, desc, stats:pseudo + └─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_char, idx_code_prefix) */ * from t_char order by code desc limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_char.code:desc, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_char.code, prefix_len:8 +└─IndexLookUp 5.00 root + ├─Limit(Build) 5.00 cop[tikv] offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_char.code, prefix_len:8 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_char, index:idx_code_prefix(code) keep order:true, desc, stats:pseudo + └─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_char keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 +└─IndexLookUp 5.00 root + ├─Limit(Build) 5.00 cop[tikv] offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_no_prefix, idx_name) */ * from t_no_prefix order by name limit 5; +id estRows task access object operator info +IndexLookUp 5.00 root limit embedded(offset:0, count:5) +├─Limit(Build) 5.00 cop[tikv] offset:0, count:5 +│ └─IndexFullScan 5.00 cop[tikv] table:t_no_prefix, index:idx_name(name) keep order:true, stats:pseudo +└─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_no_prefix keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_multi_col, idx_a_b_prefix) */ * from t_multi_col order by a, c limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_multi_col.a, planner__core__partial_order_topn.t_multi_col.c, offset:0, count:5 +└─IndexLookUp 5.00 root + ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t_multi_col, index:idx_a_b_prefix(a, b) keep order:false, stats:pseudo + └─TopN(Probe) 5.00 cop[tikv] planner__core__partial_order_topn.t_multi_col.a, planner__core__partial_order_topn.t_multi_col.c, offset:0, count:5 + └─TableRowIDScan 10000.00 cop[tikv] table:t_multi_col keep order:false, stats:pseudo +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 +└─IndexLookUp 5.00 root + ├─Limit(Build) 5.00 cop[tikv] offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t_multi_col, idx_a_b_prefix) */ * from t_multi_col order by a asc, b desc limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_multi_col.a, planner__core__partial_order_topn.t_multi_col.b:desc, offset:0, count:5 +└─IndexLookUp 5.00 root + ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t_multi_col, index:idx_a_b_prefix(a, b) keep order:false, stats:pseudo + └─TopN(Probe) 5.00 cop[tikv] planner__core__partial_order_topn.t_multi_col.a, planner__core__partial_order_topn.t_multi_col.b:desc, offset:0, count:5 + └─TableRowIDScan 10000.00 cop[tikv] table:t_multi_col keep order:false, stats:pseudo +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by upper(name) limit 5; +id estRows task access object operator info +Projection 5.00 root planner__core__partial_order_topn.t_varchar.id, planner__core__partial_order_topn.t_varchar.name, planner__core__partial_order_topn.t_varchar.data +└─TopN 5.00 root Column#4, offset:0, count:5 + └─Projection 5.00 root planner__core__partial_order_topn.t_varchar.id, planner__core__partial_order_topn.t_varchar.name, planner__core__partial_order_topn.t_varchar.data, upper(planner__core__partial_order_topn.t_varchar.name)->Column#4 + └─IndexLookUp 5.00 root + ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:false, stats:pseudo + └─TopN(Probe) 5.00 cop[tikv] upper(planner__core__partial_order_topn.t_varchar.name), offset:0, count:5 + └─TableRowIDScan 10000.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by concat(name, data) limit 5; +id estRows task access object operator info +Projection 5.00 root planner__core__partial_order_topn.t_varchar.id, planner__core__partial_order_topn.t_varchar.name, planner__core__partial_order_topn.t_varchar.data +└─TopN 5.00 root Column#4, offset:0, count:5 + └─Projection 5.00 root planner__core__partial_order_topn.t_varchar.id, planner__core__partial_order_topn.t_varchar.name, planner__core__partial_order_topn.t_varchar.data, concat(planner__core__partial_order_topn.t_varchar.name, planner__core__partial_order_topn.t_varchar.data)->Column#4 + └─IndexLookUp 5.00 root + ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:false, stats:pseudo + └─TopN(Probe) 5.00 cop[tikv] concat(planner__core__partial_order_topn.t_varchar.name, planner__core__partial_order_topn.t_varchar.data), offset:0, count:5 + └─TableRowIDScan 10000.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select /*+ force_index(t_multi_col, idx_a_b_prefix) */ * from t_multi_col order by a limit 5; +id estRows task access object operator info +IndexLookUp 5.00 root limit embedded(offset:0, count:5) +├─Limit(Build) 5.00 cop[tikv] offset:0, count:5 +│ └─IndexFullScan 5.00 cop[tikv] table:t_multi_col, index:idx_a_b_prefix(a, b) keep order:true, stats:pseudo +└─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_multi_col keep order:false, stats:pseudo +explain format='brief' select /*+ force_index(t_no_prefix, idx_name) */ * from t_no_prefix order by name limit 5; +id estRows task access object operator info +IndexLookUp 5.00 root limit embedded(offset:0, count:5) +├─Limit(Build) 5.00 cop[tikv] offset:0, count:5 +│ └─IndexFullScan 5.00 cop[tikv] table:t_no_prefix, index:idx_name(name) keep order:true, stats:pseudo +└─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_no_prefix keep order:false, stats:pseudo +drop table if exists t_join1, t_join2; +create table t_join1 (id int primary key, name varchar(100), index idx_name(name(10))); +create table t_join2 (id int primary key, ref_id int); +insert into t_join1 values (1, 'alpha'), (2, 'beta'), (3, 'gamma'); +insert into t_join2 values (1, 1), (2, 2), (3, 3); +explain format='brief' select t_join1.* from t_join1 join t_join2 on t_join1.id = t_join2.ref_id order by t_join1.name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_join1.name, offset:0, count:5 +└─HashJoin 12487.50 root inner join, equal:[eq(planner__core__partial_order_topn.t_join2.ref_id, planner__core__partial_order_topn.t_join1.id)] + ├─TableReader(Build) 9990.00 root data:Selection + │ └─Selection 9990.00 cop[tikv] not(isnull(planner__core__partial_order_topn.t_join2.ref_id)) + │ └─TableFullScan 10000.00 cop[tikv] table:t_join2 keep order:false, stats:pseudo + └─TableReader(Probe) 10000.00 root data:TableFullScan + └─TableFullScan 10000.00 cop[tikv] table:t_join1 keep order:false, stats:pseudo +explain format='brief' select name, count(*) from t_varchar group by name order by name limit 5; +id estRows task access object operator info +Projection 5.00 root planner__core__partial_order_topn.t_varchar.name, Column#4 +└─TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5 + └─HashAgg 8000.00 root group by:planner__core__partial_order_topn.t_varchar.name, funcs:count(Column#5)->Column#4, funcs:firstrow(planner__core__partial_order_topn.t_varchar.name)->planner__core__partial_order_topn.t_varchar.name + └─TableReader 8000.00 root data:HashAgg + └─HashAgg 8000.00 cop[tikv] group by:planner__core__partial_order_topn.t_varchar.name, funcs:count(1)->Column#5 + └─TableFullScan 10000.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' (select name from t_varchar order by name limit 3) union all (select name from t_no_prefix order by name limit 3) order by name limit 5; +id estRows task access object operator info +TopN 5.00 root Column#7, offset:0, count:5 +└─Union 6.00 root + ├─Limit 3.00 root offset:0, count:5 + │ └─TopN 3.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:3, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 + │ └─Projection 3.00 root planner__core__partial_order_topn.t_varchar.name + │ └─IndexLookUp 3.00 root + │ ├─Limit(Build) 3.00 cop[tikv] offset:0, count:3, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 + │ │ └─IndexFullScan 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:true, stats:pseudo + │ └─TableRowIDScan(Probe) 3.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo + └─Limit 3.00 root offset:0, count:5 + └─Limit 3.00 root offset:0, count:3 + └─IndexReader 3.00 root index:Limit + └─Limit 3.00 cop[tikv] offset:0, count:3 + └─IndexFullScan 3.00 cop[tikv] table:t_no_prefix, index:idx_name(name) keep order:true, stats:pseudo +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ id, upper(name) as upper_name from t_varchar order by upper_name limit 5; +id estRows task access object operator info +Projection 5.00 root planner__core__partial_order_topn.t_varchar.id, upper(planner__core__partial_order_topn.t_varchar.name)->Column#4 +└─Projection 5.00 root planner__core__partial_order_topn.t_varchar.id, planner__core__partial_order_topn.t_varchar.name + └─TopN 5.00 root Column#5, offset:0, count:5 + └─Projection 5.00 root planner__core__partial_order_topn.t_varchar.id, planner__core__partial_order_topn.t_varchar.name, upper(planner__core__partial_order_topn.t_varchar.name)->Column#5 + └─IndexLookUp 5.00 root + ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:false, stats:pseudo + └─TopN(Probe) 5.00 cop[tikv] upper(planner__core__partial_order_topn.t_varchar.name), offset:0, count:5 + └─TableRowIDScan 10000.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ id, length(name) as name_len from t_varchar order by name_len limit 5; +id estRows task access object operator info +Projection 5.00 root planner__core__partial_order_topn.t_varchar.id, length(planner__core__partial_order_topn.t_varchar.name)->Column#4 +└─Projection 5.00 root planner__core__partial_order_topn.t_varchar.id, planner__core__partial_order_topn.t_varchar.name + └─TopN 5.00 root Column#5, offset:0, count:5 + └─Projection 5.00 root planner__core__partial_order_topn.t_varchar.id, planner__core__partial_order_topn.t_varchar.name, length(planner__core__partial_order_topn.t_varchar.name)->Column#5 + └─IndexLookUp 5.00 root + ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:false, stats:pseudo + └─TopN(Probe) 5.00 cop[tikv] length(planner__core__partial_order_topn.t_varchar.name), offset:0, count:5 + └─TableRowIDScan 10000.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +drop table if exists t12, t15; +CREATE TABLE `t12` ( +`c0` varchar(30) DEFAULT NULL, +`c1` varchar(30) DEFAULT NULL, +`c2` varchar(30) DEFAULT NULL, +KEY `idx1` (`c0`,`c2`(10)) +); +CREATE TABLE `t15` ( +`c1` int DEFAULT NULL, +`c2` varchar(30) DEFAULT NULL, +`c3` varchar(30) DEFAULT NULL, +`c4` varchar(30) DEFAULT NULL, +KEY `idx1` (`c1`,`c2`,`c3`(10)) +); +explain format='brief' select /*+ use_index(t15, idx1) */ c1 from t15 where length(c1) > 13 order by c1, c4, c3 limit 10922 offset 55010; +id estRows task access object operator info +Projection 8000.00 root planner__core__partial_order_topn.t15.c1 +└─TopN 8000.00 root planner__core__partial_order_topn.t15.c1, planner__core__partial_order_topn.t15.c4, planner__core__partial_order_topn.t15.c3, offset:55010, count:10922 + └─IndexLookUp 8000.00 root + ├─Selection(Build) 8000.00 cop[tikv] gt(length(cast(planner__core__partial_order_topn.t15.c1, var_string(20))), 13) + │ └─IndexFullScan 10000.00 cop[tikv] table:t15, index:idx1(c1, c2, c3) keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 8000.00 cop[tikv] table:t15 keep order:false, stats:pseudo +explain format='brief' select /*+ use_index(t12, idx1) */ c0, c1 from t12 where length(c0) > 13 order by c0, c1 limit 10922 offset 55010; +id estRows task access object operator info +TopN 8000.00 root planner__core__partial_order_topn.t12.c0, planner__core__partial_order_topn.t12.c1, offset:55010, count:10922 +└─IndexLookUp 8000.00 root + ├─Selection(Build) 8000.00 cop[tikv] gt(length(planner__core__partial_order_topn.t12.c0), 13) + │ └─IndexFullScan 10000.00 cop[tikv] table:t12, index:idx1(c0, c2) keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 8000.00 cop[tikv] table:t12 keep order:false, stats:pseudo +begin; +insert into t_varchar (name, data) values ('zebra', 'animal'); +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5 +└─UnionScan 10000.00 root + └─IndexLookUp 10000.00 root + ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 10000.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +rollback; +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 +└─IndexLookUp 5.00 root + ├─Limit(Build) 5.00 cop[tikv] offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +begin; +update t_varchar set data = 'modified' where id = 1; +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5 +└─UnionScan 10000.00 root + └─IndexLookUp 10000.00 root + ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 10000.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +rollback; +begin; +delete from t_varchar where id = 1; +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5 +└─UnionScan 10000.00 root + └─IndexLookUp 10000.00 root + ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 10000.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +rollback; +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 1; +id estRows task access object operator info +TopN 1.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:1, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 +└─IndexLookUp 1.00 root + ├─Limit(Build) 1.00 cop[tikv] offset:0, count:1, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 1.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 100; +id estRows task access object operator info +TopN 100.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:100, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 +└─IndexLookUp 100.00 root + ├─Limit(Build) 100.00 cop[tikv] offset:0, count:100, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 100.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 1000; +id estRows task access object operator info +TopN 1000.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:1000, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 +└─IndexLookUp 1000.00 root + ├─Limit(Build) 1000.00 cop[tikv] offset:0, count:1000, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 1000.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +drop table if exists t_empty; +create table t_empty (id int primary key, name varchar(100), index idx_name(name(10))); +explain format='brief' select /*+ force_index(t_empty, idx_name) */ * from t_empty order by name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_empty.name, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_empty.name, prefix_len:10 +└─IndexLookUp 5.00 root + ├─Limit(Build) 5.00 cop[tikv] offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_empty.name, prefix_len:10 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_empty, index:idx_name(name) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_empty keep order:false, stats:pseudo +drop table if exists t_prefix_len; +create table t_prefix_len ( +id int primary key auto_increment, +short_prefix varchar(100), +long_prefix varchar(100), +index idx_short (short_prefix(5)), +index idx_long (long_prefix(50)) +); +insert into t_prefix_len (short_prefix, long_prefix) values +('abcdefghij', 'abcdefghijklmnopqrstuvwxyz'), +('abcdefghik', 'abcdefghijklmnopqrstuvwxya'), +('abcdefghil', 'abcdefghijklmnopqrstuvwxyb'), +('xyzdefghij', 'xyzdefghijklmnopqrstuvwxyz'), +('xyzdefghik', 'xyzdefghijklmnopqrstuvwxya'); +explain format='brief' select /*+ force_index(t_prefix_len, idx_short) */ * from t_prefix_len order by short_prefix limit 3; +id estRows task access object operator info +TopN 3.00 root planner__core__partial_order_topn.t_prefix_len.short_prefix, offset:0, count:3, prefix_col:planner__core__partial_order_topn.t_prefix_len.short_prefix, prefix_len:5 +└─IndexLookUp 3.00 root + ├─Limit(Build) 3.00 cop[tikv] offset:0, count:3, prefix_col:planner__core__partial_order_topn.t_prefix_len.short_prefix, prefix_len:5 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_prefix_len, index:idx_short(short_prefix) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 3.00 cop[tikv] table:t_prefix_len keep order:false, stats:pseudo +explain format='brief' select /*+ force_index(t_prefix_len, long_prefix) */ * from t_prefix_len order by long_prefix limit 3; +id estRows task access object operator info +TopN 3.00 root planner__core__partial_order_topn.t_prefix_len.long_prefix, offset:0, count:3, prefix_col:planner__core__partial_order_topn.t_prefix_len.long_prefix, prefix_len:50 +└─IndexLookUp 3.00 root + ├─Limit(Build) 3.00 cop[tikv] offset:0, count:3, prefix_col:planner__core__partial_order_topn.t_prefix_len.long_prefix, prefix_len:50 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_prefix_len, index:idx_long(long_prefix) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 3.00 cop[tikv] table:t_prefix_len keep order:false, stats:pseudo +set @@tidb_opt_partial_ordered_index_for_topn = 'DISABLE'; +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5 +└─IndexLookUp 5.00 root + ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:false, stats:pseudo + └─TopN(Probe) 5.00 cop[tikv] planner__core__partial_order_topn.t_varchar.name, offset:0, count:5 + └─TableRowIDScan 10000.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +set @@tidb_opt_partial_ordered_index_for_topn = 'COST'; +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 +└─IndexLookUp 5.00 root + ├─Limit(Build) 5.00 cop[tikv] offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +set @@tidb_opt_partial_ordered_index_for_topn = DEFAULT; +set @@tidb_opt_partial_ordered_index_for_topn = 'COST'; +explain format='brief' select /*+ order_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 +└─IndexLookUp 5.00 root + ├─Limit(Build) 5.00 cop[tikv] offset:0, count:5, prefix_col:planner__core__partial_order_topn.t_varchar.name, prefix_len:10 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:true, stats:pseudo + └─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +explain format='brief' select /*+ order_index(t_no_prefix, idx_name) */ * from t_no_prefix order by name limit 5; +id estRows task access object operator info +IndexLookUp 5.00 root limit embedded(offset:0, count:5) +├─Limit(Build) 5.00 cop[tikv] offset:0, count:5 +│ └─IndexFullScan 5.00 cop[tikv] table:t_no_prefix, index:idx_name(name) keep order:true, stats:pseudo +└─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_no_prefix keep order:false, stats:pseudo +explain format='brief' select /*+ order_index(t_no_prefix, idx_name) */ * from t_no_prefix order by data limit 5; +Error 1815 (HY000): Internal : Can't find a proper physical plan for this query +set @@tidb_opt_partial_ordered_index_for_topn = 'DISABLE'; +explain format='brief' select /*+ order_index(t_no_prefix, idx_name) */ * from t_no_prefix order by name limit 5; +id estRows task access object operator info +IndexLookUp 5.00 root limit embedded(offset:0, count:5) +├─Limit(Build) 5.00 cop[tikv] offset:0, count:5 +│ └─IndexFullScan 5.00 cop[tikv] table:t_no_prefix, index:idx_name(name) keep order:true, stats:pseudo +└─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_no_prefix keep order:false, stats:pseudo +explain format='brief' select /*+ order_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; +Error 1815 (HY000): Internal : Can't find a proper physical plan for this query +set @@tidb_opt_partial_ordered_index_for_topn = 'COST'; +explain format='brief' select /*+ no_order_index(t_no_prefix, idx_name) */ * from t_no_prefix order by name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_no_prefix.name, offset:0, count:5 +└─IndexLookUp 5.00 root + ├─TopN(Build) 5.00 cop[tikv] planner__core__partial_order_topn.t_no_prefix.name, offset:0, count:5 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_no_prefix, index:idx_name(name) keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_no_prefix keep order:false, stats:pseudo +explain format='brief' select /*+ no_order_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5 +└─IndexLookUp 5.00 root + ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:false, stats:pseudo + └─TopN(Probe) 5.00 cop[tikv] planner__core__partial_order_topn.t_varchar.name, offset:0, count:5 + └─TableRowIDScan 10000.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +set @@tidb_opt_partial_ordered_index_for_topn = 'DISABLE'; +explain format='brief' select /*+ no_order_index(t_no_prefix, idx_name) */ * from t_no_prefix order by name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_no_prefix.name, offset:0, count:5 +└─IndexLookUp 5.00 root + ├─TopN(Build) 5.00 cop[tikv] planner__core__partial_order_topn.t_no_prefix.name, offset:0, count:5 + │ └─IndexFullScan 10000.00 cop[tikv] table:t_no_prefix, index:idx_name(name) keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 5.00 cop[tikv] table:t_no_prefix keep order:false, stats:pseudo +explain format='brief' select /*+ no_order_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; +id estRows task access object operator info +TopN 5.00 root planner__core__partial_order_topn.t_varchar.name, offset:0, count:5 +└─IndexLookUp 5.00 root + ├─IndexFullScan(Build) 10000.00 cop[tikv] table:t_varchar, index:idx_name_prefix(name) keep order:false, stats:pseudo + └─TopN(Probe) 5.00 cop[tikv] planner__core__partial_order_topn.t_varchar.name, offset:0, count:5 + └─TableRowIDScan 10000.00 cop[tikv] table:t_varchar keep order:false, stats:pseudo +set @@tidb_opt_partial_ordered_index_for_topn = DEFAULT; +set @@tidb_opt_partial_ordered_index_for_topn = 'cost'; +select @@tidb_opt_partial_ordered_index_for_topn; +@@tidb_opt_partial_ordered_index_for_topn +COST +set @@tidb_opt_partial_ordered_index_for_topn = 'Cost'; +select @@tidb_opt_partial_ordered_index_for_topn; +@@tidb_opt_partial_ordered_index_for_topn +COST +set @@tidb_opt_partial_ordered_index_for_topn = 'COST'; +select @@tidb_opt_partial_ordered_index_for_topn; +@@tidb_opt_partial_ordered_index_for_topn +COST +set @@tidb_opt_partial_ordered_index_for_topn = 'disable'; +select @@tidb_opt_partial_ordered_index_for_topn; +@@tidb_opt_partial_ordered_index_for_topn +DISABLE +set @@tidb_opt_partial_ordered_index_for_topn = 'Disable'; +select @@tidb_opt_partial_ordered_index_for_topn; +@@tidb_opt_partial_ordered_index_for_topn +DISABLE +set @@tidb_opt_partial_ordered_index_for_topn = 'DISABLE'; +select @@tidb_opt_partial_ordered_index_for_topn; +@@tidb_opt_partial_ordered_index_for_topn +DISABLE +set @@tidb_opt_partial_ordered_index_for_topn = 'ON'; +Error 1231 (42000): Variable 'tidb_opt_partial_ordered_index_for_topn' can't be set to the value of 'ON' +set @@tidb_opt_partial_ordered_index_for_topn = 'OFF'; +Error 1231 (42000): Variable 'tidb_opt_partial_ordered_index_for_topn' can't be set to the value of 'OFF' +set @@tidb_opt_partial_ordered_index_for_topn = 1; +Error 1231 (42000): Variable 'tidb_opt_partial_ordered_index_for_topn' can't be set to the value of '1' +set @@tidb_opt_partial_ordered_index_for_topn = 0; +Error 1231 (42000): Variable 'tidb_opt_partial_ordered_index_for_topn' can't be set to the value of '0' +set @@tidb_opt_partial_ordered_index_for_topn = DEFAULT; +select @@tidb_opt_partial_ordered_index_for_topn; +@@tidb_opt_partial_ordered_index_for_topn +DISABLE +drop table if exists t_varchar, t_char, t_text, t_multi_col, t_no_prefix; +drop table if exists t_join1, t_join2, t_empty, t_prefix_len, t_null; +drop table if exists t12, t15; diff --git a/tests/integrationtest/t/planner/core/partial_order_topn.test b/tests/integrationtest/t/planner/core/partial_order_topn.test new file mode 100644 index 0000000000000..e5adafce91bb6 --- /dev/null +++ b/tests/integrationtest/t/planner/core/partial_order_topn.test @@ -0,0 +1,537 @@ +# Test cases for Partial Order TopN Optimization +# This feature allows optimizer to use prefix index for ORDER BY ... LIMIT queries +# Related PRs: #65799, #65533 + +# ========================================== +# Section 1: Table Setup (various column types) +# ========================================== + +drop table if exists t_varchar, t_char, t_text, t_multi_col, t_no_prefix; + +# Table with VARCHAR prefix index +create table t_varchar ( + id int primary key auto_increment, + name varchar(255), + data varchar(100), + index idx_name_prefix (name(10)) +); + +# Table with CHAR prefix index +create table t_char ( + id int primary key auto_increment, + code char(50), + value int, + index idx_code_prefix (code(8)) +); + +# Table with TEXT prefix index +create table t_text ( + id int primary key auto_increment, + content text, + score int, + index idx_content_prefix (content(20)) +); + +# Table with multi-column index (last column is prefix) +create table t_multi_col ( + id int primary key auto_increment, + a int, + b varchar(100), + c varchar(100), + index idx_a_b_prefix (a, b(15)), + index idx_a_b_c_prefix (a, b, c(10)) +); + +# Table without prefix index (for comparison) +create table t_no_prefix ( + id int primary key auto_increment, + name varchar(255), + data varchar(100), + index idx_name (name) +); + +# Insert test data +insert into t_varchar (name, data) values +('apple', 'fruit'), +('apricot', 'fruit'), +('application', 'software'), +('banana', 'fruit'), +('blueberry', 'fruit'), +('cherry', 'fruit'), +('date', 'fruit'), +('elderberry', 'fruit'), +('fig', 'fruit'), +('grape', 'fruit'); + +insert into t_char (code, value) values +('ABC12345', 100), +('ABC12346', 200), +('ABC12347', 300), +('DEF12345', 400), +('DEF12346', 500), +('GHI12345', 600), +('GHI12346', 700), +('JKL12345', 800), +('JKL12346', 900), +('MNO12345', 1000); + +insert into t_text (content, score) values +('This is a long text content for testing', 10), +('Another long text content here', 20), +('Some more text data for testing purposes', 30), +('Text content with different prefix', 40), +('Testing partial order optimization', 50), +('More test data with various lengths', 60), +('Short text', 70), +('Medium length text content', 80), +('Very long text content that exceeds the prefix length significantly', 90), +('Final test text content', 100); + +insert into t_multi_col (a, b, c) values +(1, 'alpha', 'first'), +(1, 'beta', 'second'), +(1, 'gamma', 'third'), +(2, 'alpha', 'fourth'), +(2, 'beta', 'fifth'), +(2, 'gamma', 'sixth'), +(3, 'alpha', 'seventh'), +(3, 'beta', 'eighth'), +(3, 'gamma', 'ninth'), +(3, 'delta', 'tenth'); + +insert into t_no_prefix (name, data) values +('apple', 'fruit'), +('apricot', 'fruit'), +('application', 'software'), +('banana', 'fruit'), +('blueberry', 'fruit'); + +# ========================================== +# Section 2: Session Variable DISABLE (baseline - no optimization) +# ========================================== + +# Verify default value is DISABLE +select @@tidb_opt_partial_ordered_index_for_topn; + +# With DISABLE, partial order optimization should NOT be used +set @@tidb_opt_partial_ordered_index_for_topn = 'DISABLE'; + +# Basic TopN with prefix index - should NOT show prefix_col/prefix_len +explain format='brief' select * from t_varchar order by name limit 5; +explain format='brief' select * from t_varchar where data = 'fruit' order by name limit 5; + +# Multi-column index - should NOT show partial order optimization +explain format='brief' select * from t_multi_col where a = 1 order by b limit 5; + +# ========================================== +# Section 3: Session Variable COST (optimization enabled) +# To ensure the test results are stable (rather than based on data volume), +# the planner testing will currently use a combination of Session Variable and Hint "use index". +# TODO: After completing the dynamic selection of the optimal plan based on the cost model, +# we will add testing for the cost model. +# ========================================== + +set @@tidb_opt_partial_ordered_index_for_topn = 'COST'; + +# Verify setting +select @@tidb_opt_partial_ordered_index_for_topn; + +# ========================================== +# Section 3.1: Supported Query Pattern - TopN -> DataSource +# ========================================== + +# Single column prefix index on VARCHAR +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; + +# Single column prefix index on CHAR +explain format='brief' select /*+ use_index(t_char, idx_code_prefix) */ * from t_char order by code limit 5; + +# Single column prefix index on TEXT +explain format='brief' select /*+ use_index(t_text, idx_content_prefix) */ * from t_text order by content limit 5; + +# ========================================== +# Section 3.2: Supported Query Pattern - TopN -> Selection -> DataSource +# ========================================== + +# Selection can be push down to Index Scan side +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ * from t_varchar where id=1 order by name limit 5; + +# Selection keep in TiDB side +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ * from t_varchar where left(name, 2) = 'ap' order by name limit 5; + +# Selection with equality condition +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ * from t_varchar where data = 'fruit' order by name limit 5; + +# Selection with range condition +explain format='brief' select /*+ use_index(t_char, idx_code_prefix) */ * from t_char where value > 300 order by code limit 5; + +# Selection with LIKE condition +explain format='brief' select /*+ use_index(t_text, idx_content_prefix) */ * from t_text where score < 80 order by content limit 3; + +# ========================================== +# Section 3.3: Supported Query Pattern - TopN -> Projection -> DataSource +# ========================================== + +# Simple projection (subquery projection) +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ id, inner_name from (select id, name as inner_name from t_varchar) tmp order by inner_name limit 5; + +# Projection with column alias +explain format='brief' select /*+ use_index(t_char, idx_code_prefix) */ id, code as c from t_char order by code limit 5; + +# ========================================== +# Section 3.4: Supported Query Pattern - TopN -> Projection -> Selection -> DataSource +# ========================================== + +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ id, name from t_varchar where data = 'fruit' order by name limit 5; +explain format='brief' select /*+ use_index(t_char, idx_code_prefix) */ id, code as c from t_char where value > 200 order by code limit 5; + +# ========================================== +# Section 3.5: Supported Query Pattern - TopN -> Selection -> Projection -> DataSource +# ========================================== + +# This pattern is also supported +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ id, name from t_varchar where data = 'fruit' order by name limit 5; + +# ========================================== +# Section 3.6: Multi-column Index with Last Column as Prefix +# ========================================== + +# ORDER BY matches prefix of multi-column index +explain format='brief' select /*+ use_index(t_multi_col, idx_a_b_prefix) */ * from t_multi_col where a = 1 order by b limit 5; + +# ORDER BY a, b with index (a, b(15)) +explain format='brief' select /*+ use_index(t_multi_col, idx_a_b_prefix) */ * from t_multi_col order by a, b limit 5; + +# ORDER BY a, b, c with index (a, b, c(10)) +explain format='brief' select /*+ use_index(t_multi_col, idx_a_b_c_prefix) */ * from t_multi_col order by a, b, c limit 5; + +# ========================================== +# Section 3.7: With OFFSET +# ========================================== + +# LIMIT with OFFSET +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5 offset 3; +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 10 offset 5; + +# ========================================== +# Section 3.8: DESC Order +# ========================================== + +# DESC order should also work +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name desc limit 5; +explain format='brief' select /*+ use_index(t_char, idx_code_prefix) */ * from t_char order by code desc limit 5; + +# ========================================== +# Section 4: USE INDEX hint + COST mode +# ========================================== + +# use_index hint with matching prefix index - should use partial order +explain format='brief' select /*+ use_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; + +# use_index hint with non-matching index - should degenerate to normal behavior +explain format='brief' select /*+ use_index(t_no_prefix, idx_name) */ * from t_no_prefix order by name limit 5; +explain format='brief' select /*+ use_index(t_multi_col, idx_a_b_prefix) */ * from t_multi_col order by a, c limit 5; + +# force_index hint with prefix index +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; + +# ========================================== +# Section 5: Unsupported patterns (should fallback to normal plan) +# ========================================== + +# ========================================== +# Section 5.1: Mixed ASC/DESC in ORDER BY +# ========================================== + +# Mixed order direction - NOT supported +explain format='brief' select /*+ use_index(t_multi_col, idx_a_b_prefix) */ * from t_multi_col order by a asc, b desc limit 5; + +# ========================================== +# Section 5.2: ORDER BY with expressions (not simple column references) +# ========================================== + +# Expression in ORDER BY - NOT supported +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by upper(name) limit 5; +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by concat(name, data) limit 5; + +# ========================================== +# Section 5.3: Index columns > ORDER BY columns +# ========================================== + +# When index has more columns than ORDER BY, partial order not applicable +# index idx_a_b_prefix (a, b(15)) but ORDER BY only a +explain format='brief' select /*+ force_index(t_multi_col, idx_a_b_prefix) */ * from t_multi_col order by a limit 5; + +# ========================================== +# Section 5.4: Non-prefix index (last column not prefix) +# ========================================== + +# Table with normal index (no prefix) - should NOT use partial order +explain format='brief' select /*+ force_index(t_no_prefix, idx_name) */ * from t_no_prefix order by name limit 5; + +# ========================================== +# Section 5.5: Unsupported plan patterns +# ========================================== + +# TopN with JOIN - NOT supported +drop table if exists t_join1, t_join2; +create table t_join1 (id int primary key, name varchar(100), index idx_name(name(10))); +create table t_join2 (id int primary key, ref_id int); +insert into t_join1 values (1, 'alpha'), (2, 'beta'), (3, 'gamma'); +insert into t_join2 values (1, 1), (2, 2), (3, 3); + +explain format='brief' select t_join1.* from t_join1 join t_join2 on t_join1.id = t_join2.ref_id order by t_join1.name limit 5; + +# TopN with Aggregation - NOT supported +explain format='brief' select name, count(*) from t_varchar group by name order by name limit 5; + +# TopN with UNION - NOT supported +explain format='brief' (select name from t_varchar order by name limit 3) union all (select name from t_no_prefix order by name limit 3) order by name limit 5; + +# ========================================== +# Section 5.6: Projection with function compute - NOT supported +# ========================================== + +# Projection with computed column +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ id, upper(name) as upper_name from t_varchar order by upper_name limit 5; +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ id, length(name) as name_len from t_varchar order by name_len limit 5; + +# ========================================== +# Section 5.7: The index column has been pruned - NOT supported +# ========================================== +drop table if exists t12, t15; +CREATE TABLE `t12` ( + `c0` varchar(30) DEFAULT NULL, + `c1` varchar(30) DEFAULT NULL, + `c2` varchar(30) DEFAULT NULL, + KEY `idx1` (`c0`,`c2`(10)) +); +CREATE TABLE `t15` ( + `c1` int DEFAULT NULL, + `c2` varchar(30) DEFAULT NULL, + `c3` varchar(30) DEFAULT NULL, + `c4` varchar(30) DEFAULT NULL, + KEY `idx1` (`c1`,`c2`,`c3`(10)) +); + +# The middle column of the index is pruned, partial order optimization cannot be applied +explain format='brief' select /*+ use_index(t15, idx1) */ c1 from t15 where length(c1) > 13 order by c1, c4, c3 limit 10922 offset 55010; +# The last column of index which is prefix column is pruned, partial order optimization cannot be applied +explain format='brief' select /*+ use_index(t12, idx1) */ c0, c1 from t12 where length(c0) > 13 order by c0, c1 limit 10922 offset 55010; + +# ========================================== +# Section 6: Dirty write scenarios (uncommitted data) +# ========================================== + +# When there are uncommitted writes in the transaction, partial order optimization +# should NOT be used because LogicalUnionScan is present + +# Start a transaction and insert uncommitted data +begin; +insert into t_varchar (name, data) values ('zebra', 'animal'); + +# Query with uncommitted data - should NOT show partial order optimization +# because LogicalUnionScan blocks the optimization +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; + +# Rollback the transaction +rollback; + +# After rollback, partial order should work again +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; + +# Test with UPDATE in transaction +begin; +update t_varchar set data = 'modified' where id = 1; + +# Query with uncommitted update - should NOT use partial order +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; + +rollback; + +# Test with DELETE in transaction +begin; +delete from t_varchar where id = 1; + +# Query with uncommitted delete - should NOT use partial order +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; + +rollback; + +# ========================================== +# Section 7: Edge cases +# ========================================== + +# ========================================== +# Section 7.1: Various LIMIT values +# ========================================== + +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 1; +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 100; +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 1000; + +# ========================================== +# Section 7.2: Empty table +# ========================================== + +drop table if exists t_empty; +create table t_empty (id int primary key, name varchar(100), index idx_name(name(10))); + +explain format='brief' select /*+ force_index(t_empty, idx_name) */ * from t_empty order by name limit 5; + +# ========================================== +# Section 7.3: Different prefix lengths +# ========================================== + +drop table if exists t_prefix_len; +create table t_prefix_len ( + id int primary key auto_increment, + short_prefix varchar(100), + long_prefix varchar(100), + index idx_short (short_prefix(5)), + index idx_long (long_prefix(50)) +); + +insert into t_prefix_len (short_prefix, long_prefix) values +('abcdefghij', 'abcdefghijklmnopqrstuvwxyz'), +('abcdefghik', 'abcdefghijklmnopqrstuvwxya'), +('abcdefghil', 'abcdefghijklmnopqrstuvwxyb'), +('xyzdefghij', 'xyzdefghijklmnopqrstuvwxyz'), +('xyzdefghik', 'xyzdefghijklmnopqrstuvwxya'); + +# Short prefix (5 bytes) +explain format='brief' select /*+ force_index(t_prefix_len, idx_short) */ * from t_prefix_len order by short_prefix limit 3; + +# Long prefix (50 bytes) +explain format='brief' select /*+ force_index(t_prefix_len, long_prefix) */ * from t_prefix_len order by long_prefix limit 3; + +# ========================================== +# Section 7.4: Verify actual query results are correct +# TODO: Verify it after executor PR merged +# ========================================== + +# Verify results with partial order optimization +# select * from t_varchar order by name limit 5; +# select * from t_char order by code limit 5; +# select * from t_multi_col where a = 1 order by b limit 3; + +# Verify DESC results +# select * from t_varchar order by name desc limit 5; + +# Verify with OFFSET +# select * from t_varchar order by name limit 3 offset 2; + +# ========================================== +# Section 8: Comparison between DISABLE and COST modes +# ========================================== + +# DISABLE mode - baseline +set @@tidb_opt_partial_ordered_index_for_topn = 'DISABLE'; +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; + +# COST mode - with optimization +set @@tidb_opt_partial_ordered_index_for_topn = 'COST'; +explain format='brief' select /*+ force_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; + +# Reset to default +set @@tidb_opt_partial_ordered_index_for_topn = DEFAULT; + +# ========================================== +# Section 9: ORDER_INDEX / NO_ORDER_INDEX with session variable combinations +# ========================================== + +# ========================================== +# Section 9.1: ORDER_INDEX + COST +# 1) choose partial-order index directly +# 2) choose normal keep-order index only +# 3) cannot keep order -> invalid task +# ========================================== + +set @@tidb_opt_partial_ordered_index_for_topn = 'COST'; + +# 1) partial order index (prefix index) is available in COST mode +explain format='brief' select /*+ order_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; + +# 2) choose normal keep-order index +explain format='brief' select /*+ order_index(t_no_prefix, idx_name) */ * from t_no_prefix order by name limit 5; + +# 3) cannot satisfy keep order requirement +-- error 1815 +explain format='brief' select /*+ order_index(t_no_prefix, idx_name) */ * from t_no_prefix order by data limit 5; + +# ========================================== +# Section 9.2: ORDER_INDEX + DISABLE +# 1) normal keep-order index still works +# 2) cannot keep order -> invalid task +# ========================================== + +set @@tidb_opt_partial_ordered_index_for_topn = 'DISABLE'; + +# 1) normal keep-order index +explain format='brief' select /*+ order_index(t_no_prefix, idx_name) */ * from t_no_prefix order by name limit 5; + +# 2) prefix index cannot be used as keep-order index in DISABLE mode +-- error 1815 +explain format='brief' select /*+ order_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; + +# ========================================== +# Section 9.3: NO_ORDER_INDEX + COST / DISABLE +# ========================================== + +set @@tidb_opt_partial_ordered_index_for_topn = 'COST'; +explain format='brief' select /*+ no_order_index(t_no_prefix, idx_name) */ * from t_no_prefix order by name limit 5; +explain format='brief' select /*+ no_order_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; + +set @@tidb_opt_partial_ordered_index_for_topn = 'DISABLE'; +explain format='brief' select /*+ no_order_index(t_no_prefix, idx_name) */ * from t_no_prefix order by name limit 5; +explain format='brief' select /*+ no_order_index(t_varchar, idx_name_prefix) */ * from t_varchar order by name limit 5; + +# Reset to default +set @@tidb_opt_partial_ordered_index_for_topn = DEFAULT; + +# ========================================== +# Section 10: Case sensitivity of session variable +# ========================================== + +set @@tidb_opt_partial_ordered_index_for_topn = 'cost'; +select @@tidb_opt_partial_ordered_index_for_topn; + +set @@tidb_opt_partial_ordered_index_for_topn = 'Cost'; +select @@tidb_opt_partial_ordered_index_for_topn; + +set @@tidb_opt_partial_ordered_index_for_topn = 'COST'; +select @@tidb_opt_partial_ordered_index_for_topn; + +set @@tidb_opt_partial_ordered_index_for_topn = 'disable'; +select @@tidb_opt_partial_ordered_index_for_topn; + +set @@tidb_opt_partial_ordered_index_for_topn = 'Disable'; +select @@tidb_opt_partial_ordered_index_for_topn; + +set @@tidb_opt_partial_ordered_index_for_topn = 'DISABLE'; +select @@tidb_opt_partial_ordered_index_for_topn; + +# Invalid values should error +--error 1231 +set @@tidb_opt_partial_ordered_index_for_topn = 'ON'; + +--error 1231 +set @@tidb_opt_partial_ordered_index_for_topn = 'OFF'; + +--error 1231 +set @@tidb_opt_partial_ordered_index_for_topn = 1; + +--error 1231 +set @@tidb_opt_partial_ordered_index_for_topn = 0; + +# Reset to default +set @@tidb_opt_partial_ordered_index_for_topn = DEFAULT; +select @@tidb_opt_partial_ordered_index_for_topn; + +# ========================================== +# Cleanup +# ========================================== + +drop table if exists t_varchar, t_char, t_text, t_multi_col, t_no_prefix; +drop table if exists t_join1, t_join2, t_empty, t_prefix_len, t_null; +drop table if exists t12, t15;