diff --git a/pkg/executor/test/simpletest/BUILD.bazel b/pkg/executor/test/simpletest/BUILD.bazel index d7588bbba545b..c836c3c4de752 100644 --- a/pkg/executor/test/simpletest/BUILD.bazel +++ b/pkg/executor/test/simpletest/BUILD.bazel @@ -9,7 +9,7 @@ go_test( ], flaky = True, race = "on", - shard_count = 12, + shard_count = 13, deps = [ "//pkg/config", "//pkg/errno", diff --git a/pkg/parser/ast/BUILD.bazel b/pkg/parser/ast/BUILD.bazel index 14898f8435299..5fd5288e87290 100644 --- a/pkg/parser/ast/BUILD.bazel +++ b/pkg/parser/ast/BUILD.bazel @@ -45,6 +45,7 @@ go_test( "functions_test.go", "misc_test.go", "procedure_test.go", + "stats_test.go", "util_test.go", ], embed = [":ast"], diff --git a/pkg/planner/BUILD.bazel b/pkg/planner/BUILD.bazel index 3dc6c56fef4c2..8a7858e9e9a25 100644 --- a/pkg/planner/BUILD.bazel +++ b/pkg/planner/BUILD.bazel @@ -17,6 +17,7 @@ go_library( "//pkg/planner/core", "//pkg/planner/core/base", "//pkg/planner/core/resolve", + "//pkg/planner/core/rule", "//pkg/planner/indexadvisor", "//pkg/planner/planctx", "//pkg/planner/property", @@ -24,6 +25,7 @@ go_library( "//pkg/planner/util/optimizetrace", "//pkg/privilege", "//pkg/sessionctx", + "//pkg/sessionctx/stmtctx", "//pkg/sessionctx/variable", "//pkg/types", "//pkg/util", diff --git a/pkg/planner/core/casetest/correlated/BUILD.bazel b/pkg/planner/core/casetest/correlated/BUILD.bazel index bd738c8dbab48..d9cf469de1023 100644 --- a/pkg/planner/core/casetest/correlated/BUILD.bazel +++ b/pkg/planner/core/casetest/correlated/BUILD.bazel @@ -8,9 +8,11 @@ go_test( "main_test.go", ], flaky = True, + shard_count = 3, deps = [ "//pkg/testkit", "//pkg/testkit/testsetup", + "@com_github_stretchr_testify//require", "@org_uber_go_goleak//:goleak", ], ) diff --git a/pkg/planner/core/casetest/correlated/correlated_test.go b/pkg/planner/core/casetest/correlated/correlated_test.go index e331bb1cfbe2e..6ef646a4f3887 100644 --- a/pkg/planner/core/casetest/correlated/correlated_test.go +++ b/pkg/planner/core/casetest/correlated/correlated_test.go @@ -15,9 +15,12 @@ package correlated import ( + "fmt" + "strings" "testing" "github.com/pingcap/tidb/pkg/testkit" + "github.com/stretchr/testify/require" ) func TestCorrelatedSubquery(t *testing.T) { @@ -71,6 +74,78 @@ WHERE NOT (tlc07c2a51.col_1>= HAVING tlc07c2a51.col_6>0)) ;`).Check(testkit.Rows("1", "1", "1", "1", "1", "1", "1", "1", "1", "1")) } +func TestNaturalJoinWithCorrelatedSubquery(tt *testing.T) { + testkit.RunTestUnderCascades(tt, func(t *testing.T, tk *testkit.TestKit, cascades, caller string) { + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int)") + // Keep duplicate and NULL rows so the regression also pins multiplicity + // and NULL-handling for the correlated EXISTS predicate. + tk.MustExec("insert into t values (1), (1), (2), (null)") + + sql := `select t1.a + from t t1 natural join t t2 + where exists (select 1 from t t3 where t3.a = t1.a) + order by t1.a` + tk.MustQuery(sql).Check(testkit.Rows("1", "1", "1", "1", "2")) + + if cascades == "on" { + return + } + + t.Run("AlternativeLogicalPlansChooseApply", func(t *testing.T) { + tk.MustExec("use test") + tk.MustExec("drop table if exists alt_pick_t1, alt_pick_t2, alt_pick_t3") + tk.MustExec("create table alt_pick_t1(a int primary key)") + tk.MustExec("create table alt_pick_t2(a int, b int, key idx_a(a))") + tk.MustExec("create table alt_pick_t3(a int, c int, key idx_a(a))") + tk.MustExec("insert into alt_pick_t1 values (1), (2)") + + vals2 := make([]string, 0, 200) + vals3 := make([]string, 0, 200) + for i := 0; i < 200; i++ { + vals2 = append(vals2, fmt.Sprintf("(%d, %d)", i%100, i)) + vals3 = append(vals3, fmt.Sprintf("(%d, %d)", i%100, i)) + } + tk.MustExec("insert into alt_pick_t2 values " + strings.Join(vals2, ",")) + tk.MustExec("insert into alt_pick_t3 values " + strings.Join(vals3, ",")) + tk.MustExec("analyze table alt_pick_t1, alt_pick_t2, alt_pick_t3") + + tk.MustExec("set @@tidb_opt_enable_alternative_logical_plans=off") + sql := "select alt_pick_t1.a, (select count(*) from alt_pick_t2 join alt_pick_t3 on alt_pick_t2.a = alt_pick_t3.a where alt_pick_t2.a = alt_pick_t1.a) as cnt from alt_pick_t1 order by alt_pick_t1.a" + explainSQL := "explain format = 'brief' " + sql + + offPlan := tk.MustQuery(explainSQL).Rows() + tk.MustQuery(sql).Check(testkit.Rows("1 4", "2 4")) + require.False(t, rowsContainText(offPlan, "Apply"), rowsText(offPlan)) + + tk.MustExec("set @@tidb_opt_enable_alternative_logical_plans=on") + onPlan := tk.MustQuery(explainSQL).Rows() + tk.MustQuery(sql).Check(testkit.Rows("1 4", "2 4")) + require.True(t, rowsContainText(onPlan, "Apply"), rowsText(onPlan)) + }) + + t.Run("AlternativeLogicalPlansSkipSecondRoundWhenIndexJoinExists", func(t *testing.T) { + tk.MustExec("use test") + tk.MustExec("set @@tidb_opt_enable_alternative_logical_plans=on") + tk.MustExec("drop table if exists alt_skip_t1, alt_skip_t2") + tk.MustExec("create table alt_skip_t1(a int primary key)") + tk.MustExec("create table alt_skip_t2(a int, b int, key idx_a(a))") + tk.MustExec("insert into alt_skip_t1 values (1), (2), (3)") + tk.MustExec("insert into alt_skip_t2 values (1, 1), (1, 2), (2, 3), (3, 4)") + tk.MustExec("analyze table alt_skip_t1, alt_skip_t2") + + sql := "select alt_skip_t1.a from alt_skip_t1 where exists (select 1 from alt_skip_t2 where alt_skip_t2.a = alt_skip_t1.a and alt_skip_t2.b > 0) order by alt_skip_t1.a" + plan := tk.MustQuery("explain format = 'brief' " + sql).Rows() + require.False(t, rowsContainText(plan, "Apply"), rowsText(plan)) + tk.MustQuery(sql).Check(testkit.Rows("1", "2", "3")) + stmtCtx := tk.Session().GetSessionVars().StmtCtx + require.True(t, stmtCtx.AlternativeLogicalPlanDecorrelatedApply) + require.True(t, stmtCtx.AlternativeLogicalPlanSameOrderIndexJoin) + }) + }) +} + func TestWrongDecorrelate(t *testing.T) { store := testkit.CreateMockStore(t) tk := testkit.NewTestKit(t, store) @@ -84,3 +159,26 @@ func TestWrongDecorrelate(t *testing.T) { " 30025.20000000000000000000 60121022342", "X 6.23000000000000000000 60021022342")) } + +func rowsContainText(rows [][]any, needle string) bool { + for _, row := range rows { + for _, col := range row { + if strings.Contains(fmt.Sprint(col), needle) { + return true + } + } + } + return false +} + +func rowsText(rows [][]any) string { + lines := make([]string, 0, len(rows)) + for _, row := range rows { + vals := make([]string, 0, len(row)) + for _, col := range row { + vals = append(vals, fmt.Sprint(col)) + } + lines = append(lines, strings.Join(vals, " ")) + } + return strings.Join(lines, "\n") +} diff --git a/pkg/planner/core/exhaust_physical_plans.go b/pkg/planner/core/exhaust_physical_plans.go index d999bcdf9973a..268a87da761d9 100644 --- a/pkg/planner/core/exhaust_physical_plans.go +++ b/pkg/planner/core/exhaust_physical_plans.go @@ -567,6 +567,8 @@ func constructIndexJoin( CompareFilters: compareFilters, OuterHashKeys: outerHashKeys, InnerHashKeys: innerHashKeys, + // Only count candidates that keep the original Apply outer/inner order. + FromDecorrelatedApply: p.FromDecorrelatedApply && outerIdx == 0, }.Init(p.SCtx(), p.StatsInfo().ScaleByExpectCnt(prop.ExpectedCnt), p.QueryBlockOffset(), chReqProps...) if path != nil { join.IdxColLens = path.IdxColLens diff --git a/pkg/planner/core/logical_plan_builder.go b/pkg/planner/core/logical_plan_builder.go index ead149d7bdf80..b4bb77558c860 100644 --- a/pkg/planner/core/logical_plan_builder.go +++ b/pkg/planner/core/logical_plan_builder.go @@ -3749,7 +3749,7 @@ func (b *PlanBuilder) pushHintWithoutTableWarning(hint *ast.TableOptimizerHint) } func (b *PlanBuilder) pushTableHints(hints []*ast.TableOptimizerHint, currentLevel int) { - hints = b.hintProcessor.GetCurrentStmtHints(hints, currentLevel) + hints = b.hintProcessor.GetCurrentStmtHints(hints, currentLevel, b.hintState) sessionVars := b.ctx.GetSessionVars() currentDB := sessionVars.CurrentDB warnHandler := sessionVars.StmtCtx @@ -4628,7 +4628,7 @@ func (b *PlanBuilder) buildDataSource(ctx context.Context, tn *ast.TableName, as // Because of the nested views, so we should check the left table list in hint when build the data source from the view inside the current view. currentQBNameMap4View[qbName] = viewQBNameHintTable[1:] currentViewHints[qbName] = b.hintProcessor.ViewQBNameToHints[qbName] - b.hintProcessor.ViewQBNameUsed[qbName] = struct{}{} + b.hintProcessor.MarkViewQBNameUsed(qbName, b.hintState) } } return b.BuildDataSourceFromView(ctx, dbName, tableInfo, currentQBNameMap4View, currentViewHints) @@ -5109,18 +5109,21 @@ func (b *PlanBuilder) BuildDataSourceFromView(ctx context.Context, dbName pmodel hintProcessor.ViewQBNameToTable = qbNameMap4View hintProcessor.ViewQBNameToHints = viewHints - hintProcessor.ViewQBNameUsed = make(map[string]struct{}) - hintProcessor.QBOffsetToHints = currentQbHints hintProcessor.QBNameToSelOffset = currentQbNameMap + hintState := hintProcessor.NewBuildState() + hintState.QBOffsetToHints = currentQbHints originHintProcessor := b.hintProcessor + originHintState := b.hintState originPlannerSelectBlockAsName := b.ctx.GetSessionVars().PlannerSelectBlockAsName.Load() b.hintProcessor = hintProcessor + b.hintState = hintState newPlannerSelectBlockAsName := make([]ast.HintTable, hintProcessor.MaxSelectStmtOffset()+1) b.ctx.GetSessionVars().PlannerSelectBlockAsName.Store(&newPlannerSelectBlockAsName) defer func() { - b.hintProcessor.HandleUnusedViewHints() + b.hintProcessor.SetWarns(b.hintProcessor.HandleUnusedViewHints(b.hintState, nil)) b.hintProcessor = originHintProcessor + b.hintState = originHintState b.ctx.GetSessionVars().PlannerSelectBlockAsName.Store(originPlannerSelectBlockAsName) }() nodeW := resolve.NewNodeWWithCtx(selectNode, b.resolveCtx) diff --git a/pkg/planner/core/operator/logicalop/logical_join.go b/pkg/planner/core/operator/logicalop/logical_join.go index 8741ffd9fdaad..7d808b247d976 100644 --- a/pkg/planner/core/operator/logicalop/logical_join.go +++ b/pkg/planner/core/operator/logicalop/logical_join.go @@ -146,6 +146,11 @@ type LogicalJoin struct { // EqualCondOutCnt indicates the estimated count of joined rows after evaluating `EqualConditions`. EqualCondOutCnt float64 + + // FromDecorrelatedApply marks joins that come from decorrelating an Apply in the + // first logical round. It is only used to decide whether an equivalent same-order + // PhysicalIndexJoin candidate has already been generated. + FromDecorrelatedApply bool } // Init initializes LogicalJoin. diff --git a/pkg/planner/core/physical_plans.go b/pkg/planner/core/physical_plans.go index ca5622b9cb216..5c96a984e4fea 100644 --- a/pkg/planner/core/physical_plans.go +++ b/pkg/planner/core/physical_plans.go @@ -1714,6 +1714,9 @@ type PhysicalIndexJoin struct { // InnerHashKeys indicates the inner keys used to build hash table during // execution. InnerJoinKeys is the prefix of InnerHashKeys. InnerHashKeys []*expression.Column + // FromDecorrelatedApply is true only when this IndexJoin keeps the original + // Apply outer/inner order after decorrelation. + FromDecorrelatedApply bool } // Clone implements op.PhysicalPlan interface. @@ -1737,6 +1740,7 @@ func (p *PhysicalIndexJoin) Clone(newCtx base.PlanContext) (base.PhysicalPlan, e cloned.CompareFilters = p.CompareFilters.Copy() cloned.OuterHashKeys = util.CloneCols(p.OuterHashKeys) cloned.InnerHashKeys = util.CloneCols(p.InnerHashKeys) + cloned.FromDecorrelatedApply = p.FromDecorrelatedApply return cloned, nil } diff --git a/pkg/planner/core/planbuilder.go b/pkg/planner/core/planbuilder.go index 09d433dbb9967..c909b6bab29c6 100644 --- a/pkg/planner/core/planbuilder.go +++ b/pkg/planner/core/planbuilder.go @@ -256,7 +256,10 @@ type PlanBuilder struct { // finish building the subquery or CTE. handleHelper *handleColHelper + // read-only meta derived from ast node. hintProcessor *hint.QBHintHandler + // mutable state of QBHint when building. + hintState *hint.QBHintBuildState // qbOffset is the offsets of current processing select stmts. qbOffset []int @@ -405,6 +408,11 @@ func GetDBTableInfo(visitInfo []visitInfo) []stmtctx.TableEntry { return tables } +// GetHintState gets the HintState from the PlanBuilder. +func (b *PlanBuilder) GetHintState() *hint.QBHintBuildState { + return b.hintState +} + // GetOptFlag gets the OptFlag of the PlanBuilder. func (b *PlanBuilder) GetOptFlag() uint64 { if b.isSampling { @@ -482,6 +490,9 @@ func (b *PlanBuilder) Init(sctx base.PlanContext, is infoschema.InfoSchema, proc b.ctx = sctx b.is = is b.hintProcessor = processor + if processor != nil { + b.hintState = processor.NewBuildState() + } b.isForUpdateRead = sctx.GetSessionVars().IsPessimisticReadConsistency() b.noDecorrelate = sctx.GetSessionVars().EnableNoDecorrelateInSelect if savedBlockNames == nil { @@ -523,6 +534,14 @@ func (b *PlanBuilder) ResetForReuse() *PlanBuilder { return b } +// HandleUnusedViewHints appends warnings for unused view hints in the current build. +func (b *PlanBuilder) HandleUnusedViewHints() { + if b.hintProcessor == nil { + return + } + b.hintProcessor.SetWarns(b.hintProcessor.HandleUnusedViewHints(b.hintState, nil)) +} + // Build builds the ast node to a Plan. func (b *PlanBuilder) Build(ctx context.Context, node *resolve.NodeW) (base.Plan, error) { // Build might be called recursively, right now they all share the same resolve diff --git a/pkg/planner/core/rule_decorrelate.go b/pkg/planner/core/rule_decorrelate.go index ab2d8c0f031c0..9cf49920e520d 100644 --- a/pkg/planner/core/rule_decorrelate.go +++ b/pkg/planner/core/rule_decorrelate.go @@ -145,6 +145,10 @@ func (s *DecorrelateSolver) Optimize(ctx context.Context, p base.LogicalPlan, op join := &apply.LogicalJoin join.SetSelf(join) join.SetTP(plancodec.TypeJoin) + if p.SCtx().GetSessionVars().EnableAlternativeLogicalPlans { + p.SCtx().GetSessionVars().StmtCtx.MarkAlternativeLogicalPlanDecorrelatedApply() + join.FromDecorrelatedApply = true + } p = join appendApplySimplifiedTraceStep(apply, join, opt) } else if apply.NoDecorrelate { diff --git a/pkg/planner/core/task.go b/pkg/planner/core/task.go index 02d9f398b293c..53c249223f409 100644 --- a/pkg/planner/core/task.go +++ b/pkg/planner/core/task.go @@ -167,6 +167,9 @@ func (p *PhysicalIndexHashJoin) Attach2Task(tasks ...base.Task) base.Task { // Attach2Task implements PhysicalPlan interface. func (p *PhysicalIndexJoin) Attach2Task(tasks ...base.Task) base.Task { outerTask := tasks[1-p.InnerChildIdx].ConvertToRootTask(p.SCtx()) + if p.FromDecorrelatedApply { + p.SCtx().GetSessionVars().StmtCtx.MarkAlternativeLogicalPlanSameOrderIndexJoin() + } if p.InnerChildIdx == 1 { p.SetChildren(outerTask.Plan(), p.innerPlan) } else { diff --git a/pkg/planner/optimize.go b/pkg/planner/optimize.go index 8f19e77467d85..400c946cfa4ed 100644 --- a/pkg/planner/optimize.go +++ b/pkg/planner/optimize.go @@ -34,6 +34,7 @@ import ( "github.com/pingcap/tidb/pkg/planner/core" "github.com/pingcap/tidb/pkg/planner/core/base" "github.com/pingcap/tidb/pkg/planner/core/resolve" + "github.com/pingcap/tidb/pkg/planner/core/rule" "github.com/pingcap/tidb/pkg/planner/indexadvisor" "github.com/pingcap/tidb/pkg/planner/planctx" "github.com/pingcap/tidb/pkg/planner/property" @@ -41,6 +42,7 @@ import ( "github.com/pingcap/tidb/pkg/planner/util/optimizetrace" "github.com/pingcap/tidb/pkg/privilege" "github.com/pingcap/tidb/pkg/sessionctx" + "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/types" tidbutil "github.com/pingcap/tidb/pkg/util" @@ -466,9 +468,133 @@ var planBuilderPool = sync.Pool{ }, } +type logicalPlanBuildCtx struct { + stmtCtxState stmtctx.LogicalPlanBuildState + plannerSelectBlockAsName *[]ast.HintTable + mapScalarSubQ []any + mapHashCode2UniqueID map[string]int + rewritePhaseInfo variable.RewritePhaseInfo +} + +func saveLogicalPlanBuildCtx(sessVars *variable.SessionVars) logicalPlanBuildCtx { + return logicalPlanBuildCtx{ + stmtCtxState: sessVars.StmtCtx.SaveLogicalPlanBuildState(), + plannerSelectBlockAsName: sessVars.PlannerSelectBlockAsName.Load(), + mapScalarSubQ: sessVars.MapScalarSubQ, + mapHashCode2UniqueID: sessVars.MapHashCode2UniqueID4ExtendedCol, + rewritePhaseInfo: sessVars.RewritePhaseInfo, + } +} + +func restoreLogicalPlanBuildCtx(sessVars *variable.SessionVars, logicalPlanCtx logicalPlanBuildCtx) { + sessVars.StmtCtx.RestoreLogicalPlanBuildState(logicalPlanCtx.stmtCtxState) + sessVars.PlannerSelectBlockAsName.Store(logicalPlanCtx.plannerSelectBlockAsName) + sessVars.MapScalarSubQ = logicalPlanCtx.mapScalarSubQ + sessVars.MapHashCode2UniqueID4ExtendedCol = logicalPlanCtx.mapHashCode2UniqueID + sessVars.RewritePhaseInfo = logicalPlanCtx.rewritePhaseInfo +} + +func buildAndOptimizeLogicalPlanRound( + ctx context.Context, + sctx planctx.PlanContext, + node *resolve.NodeW, + is infoschema.InfoSchema, + hintProcessor *hint.QBHintHandler, + checked *bool, + optimizeStarted *bool, + beginOpt *time.Time, + needRestoreLogicalPlanCtx bool, + bestPlan *base.PhysicalPlan, + bestNames *types.NameSlice, + bestCost *float64, + bestLogicalPlanCtx *logicalPlanBuildCtx, + optFlagAdjust func(uint64) uint64, +) (base.Plan, types.NameSlice, bool, error) { + builder := planBuilderPool.Get().(*core.PlanBuilder) + defer planBuilderPool.Put(builder.ResetForReuse()) + // TODO: when buildRound > 1, only emit unused view-hint warnings for the winner build. + defer builder.HandleUnusedViewHints() + + builder.Init(sctx, is, hintProcessor) + + // todo: you can customize each round's special builder (like semi join rewrite or not by signal) + p, err := buildLogicalPlan(ctx, sctx, node, builder) + if err != nil { + return nil, nil, false, err + } + names := p.OutputNames() + + if !*checked { + // Keep privilege and lock checks fail-fast. These depend on visitInfo + // produced by the logical build, but not on the later cost winner. + if pm := privilege.GetPrivilegeManager(sctx); pm != nil { + visitInfo := core.VisitInfo4PrivCheck(ctx, is, node.Node, builder.GetVisitInfo()) + if err := core.CheckPrivilege(sctx.GetSessionVars().ActiveRoles, pm, visitInfo); err != nil { + return nil, nil, false, err + } + } + + if err := core.CheckTableLock(sctx, is, builder.GetVisitInfo()); err != nil { + return nil, nil, false, err + } + + if err := core.CheckTableMode(node); err != nil { + return nil, nil, false, err + } + *checked = true + } + + // Handle the non-logical plan statement. + logic, isLogicalPlan := p.(base.LogicalPlan) + if !isLogicalPlan { + return p, names, true, nil + } + + core.RecheckCTE(logic) + + // todo: also you can customize each round's special logical opt flag here (like decorrelate rule or not) + if !*optimizeStarted { + *optimizeStarted = true + *beginOpt = time.Now() + } + var ( + finalPlan base.PhysicalPlan + cost float64 + optErr error + ) + optFlag := builder.GetOptFlag() + if optFlagAdjust != nil { + optFlag = optFlagAdjust(optFlag) + } + if sctx.GetSessionVars().GetEnableCascadesPlanner() { + finalPlan, cost, optErr = cascades.DefaultOptimizer.FindBestPlan(sctx, logic) + } else { + finalPlan, cost, optErr = core.DoOptimize(ctx, sctx, optFlag, logic) + } + if optErr != nil { + return nil, nil, false, optErr + } + + if *bestPlan == nil || cost < *bestCost { + *bestCost = cost + *bestPlan = finalPlan + *bestNames = names + if needRestoreLogicalPlanCtx { + *bestLogicalPlanCtx = saveLogicalPlanBuildCtx(sctx.GetSessionVars()) + } + } + return p, names, false, nil +} + // optimizeCnt is a global variable only used for test. var optimizeCnt int +func shouldTryAlternativeLogicalPlanRound(sessVars *variable.SessionVars) bool { + return sessVars.EnableAlternativeLogicalPlans && + sessVars.StmtCtx.AlternativeLogicalPlanDecorrelatedApply && + !sessVars.StmtCtx.AlternativeLogicalPlanSameOrderIndexJoin +} + func optimize(ctx context.Context, sctx planctx.PlanContext, node *resolve.NodeW, is infoschema.InfoSchema) (base.Plan, types.NameSlice, float64, error) { failpoint.Inject("checkOptimizeCountOne", func(val failpoint.Value) { // only count the optimization for SQL with specified text @@ -488,60 +614,98 @@ func optimize(ctx context.Context, sctx planctx.PlanContext, node *resolve.NodeW debugtrace.EnterContextCommon(sctx) defer debugtrace.LeaveContextCommon(sctx) } + var ( + beginOpt time.Time + optimizeStarted bool + ) + defer func() { + if optimizeStarted { + sessVars.DurationOptimization = time.Since(beginOpt) + } + }() - // build logical plan + // Build the logical plan from the raw AST. The hint processor only keeps + // AST-derived metadata; per-build state is allocated inside PlanBuilder. hintProcessor := hint.NewQBHintHandler(sctx.GetSessionVars().StmtCtx) node.Node.Accept(hintProcessor) - defer hintProcessor.HandleUnusedViewHints() - builder := planBuilderPool.Get().(*core.PlanBuilder) - defer planBuilderPool.Put(builder.ResetForReuse()) - builder.Init(sctx, is, hintProcessor) - p, err := buildLogicalPlan(ctx, sctx, node, builder) + + // build multi logical plan from raw AST. + var ( + needRestoreLogicalPlanCtx = sessVars.EnableAlternativeLogicalPlans + bestCost = math.MaxFloat64 + bestPlan base.PhysicalPlan + bestNames types.NameSlice + bestLogicalPlanCtx logicalPlanBuildCtx + checked bool + ) + var initialLogicalPlanCtx logicalPlanBuildCtx + if needRestoreLogicalPlanCtx { + initialLogicalPlanCtx = saveLogicalPlanBuildCtx(sessVars) + sessVars.StmtCtx.ResetAlternativeLogicalPlanSignals() + } + + p, names, nonLogical, err := buildAndOptimizeLogicalPlanRound( + ctx, + sctx, + node, + is, + hintProcessor, + &checked, + &optimizeStarted, + &beginOpt, + needRestoreLogicalPlanCtx, + &bestPlan, + &bestNames, + &bestCost, + &bestLogicalPlanCtx, + nil, + ) if err != nil { return nil, nil, 0, err } - - activeRoles := sessVars.ActiveRoles - // Check privilege. Maybe it's better to move this to the Preprocess, but - // we need the table information to check privilege, which is collected - // into the visitInfo in the logical plan builder. - if pm := privilege.GetPrivilegeManager(sctx); pm != nil { - visitInfo := core.VisitInfo4PrivCheck(ctx, is, node.Node, builder.GetVisitInfo()) - if err := core.CheckPrivilege(activeRoles, pm, visitInfo); err != nil { - return nil, nil, 0, err - } + if nonLogical { + // keep compatible with the old. + return p, names, 0, nil } - if err := core.CheckTableLock(sctx, is, builder.GetVisitInfo()); err != nil { - return nil, nil, 0, err - } + if shouldTryAlternativeLogicalPlanRound(sessVars) { + restoreLogicalPlanBuildCtx(sessVars, initialLogicalPlanCtx) + failpoint.Inject("failIfAlternativeLogicalPlanRoundTriggered", func(val failpoint.Value) { + if testSQL, ok := val.(string); ok && testSQL == node.Node.OriginalText() { + failpoint.Return(nil, nil, 0, errors.New("unexpected alternative logical plan round")) + } + }) - if err := core.CheckTableMode(node); err != nil { - return nil, nil, 0, err + p, names, nonLogical, err = buildAndOptimizeLogicalPlanRound( + ctx, + sctx, + node, + is, + hintProcessor, + &checked, + &optimizeStarted, + &beginOpt, + needRestoreLogicalPlanCtx, + &bestPlan, + &bestNames, + &bestCost, + &bestLogicalPlanCtx, + func(flag uint64) uint64 { return flag &^ rule.FlagDecorrelate }, + ) + if err != nil { + return nil, nil, 0, err + } + if nonLogical { + return p, names, 0, nil + } } - - names := p.OutputNames() - - // Handle the non-logical plan statement. - logic, isLogicalPlan := p.(base.LogicalPlan) - if !isLogicalPlan { - return p, names, 0, nil + if bestPlan == nil { + return nil, nil, 0, errors.New("failed to build logical plan") } - - core.RecheckCTE(logic) - - // Handle the logical plan statement, use cascades planner if enabled. - if sessVars.GetEnableCascadesPlanner() { - finalPlan, cost, err := cascades.DefaultOptimizer.FindBestPlan(sctx, logic) - return finalPlan, names, cost, err + if needRestoreLogicalPlanCtx { + restoreLogicalPlanBuildCtx(sessVars, bestLogicalPlanCtx) } - - beginOpt := time.Now() - finalPlan, cost, err := core.DoOptimize(ctx, sctx, builder.GetOptFlag(), logic) - // TODO: capture plan replayer here if it matches sql and plan digest - - sessVars.DurationOptimization = time.Since(beginOpt) - return finalPlan, names, cost, err + return bestPlan, bestNames, bestCost, nil } // OptimizeExecStmt to handle the "execute" statement diff --git a/pkg/sessionctx/stmtctx/BUILD.bazel b/pkg/sessionctx/stmtctx/BUILD.bazel index 3f45049846d15..0bdd8d035d4aa 100644 --- a/pkg/sessionctx/stmtctx/BUILD.bazel +++ b/pkg/sessionctx/stmtctx/BUILD.bazel @@ -45,10 +45,13 @@ go_test( ], embed = [":stmtctx"], flaky = True, - shard_count = 14, + shard_count = 16, deps = [ "//pkg/errctx", "//pkg/kv", + "//pkg/meta/model", + "//pkg/parser/ast", + "//pkg/parser/model", "//pkg/sessionctx/variable", "//pkg/testkit", "//pkg/testkit/testfailpoint", diff --git a/pkg/sessionctx/stmtctx/stmtctx.go b/pkg/sessionctx/stmtctx/stmtctx.go index 7e3feeb3732f0..1fd1587f6c919 100644 --- a/pkg/sessionctx/stmtctx/stmtctx.go +++ b/pkg/sessionctx/stmtctx/stmtctx.go @@ -71,6 +71,25 @@ type jsonSQLWarn struct { Msg string `json:"msg,omitempty"` } +// LogicalPlanBuildState stores the statement-scoped planner state that is mutated while +// building a logical plan from AST. +type LogicalPlanBuildState struct { + warnings []SQLWarn + extraWarnings []SQLWarn + tables []TableEntry + tableStats map[int64]any + lockTableIDs map[int64]struct{} + tblInfo2UnionScan map[*model.TableInfo]bool + useDynamicPruneMode bool + colRefFromUpdatePlan intset.FastIntSet + // plan cache related stuff + planCacheUseCache bool + planCacheType contextutil.PlanCacheType + planCacheUnqualified string + planCacheForce bool + planCacheAlwaysWarn bool +} + // ReferenceCount indicates the reference count of StmtCtx. type ReferenceCount int32 @@ -446,6 +465,12 @@ type StatementContext struct { UseDynamicPruneMode bool // ColRefFromPlan mark the column ref used by assignment in update statement. ColRefFromUpdatePlan intset.FastIntSet + // AlternativeLogicalPlanDecorrelatedApply indicates whether the current logical + // optimization round decorrelated at least one Apply into Join. + AlternativeLogicalPlanDecorrelatedApply bool + // AlternativeLogicalPlanSameOrderIndexJoin indicates whether the current first + // round already produced a same-order index join candidate for a decorrelated Apply. + AlternativeLogicalPlanSameOrderIndexJoin bool // IsExplainAnalyzeDML is true if the statement is "explain analyze DML executors", before responding the explain // results to the client, the transaction should be committed first. See issue #37373 for more details. @@ -572,6 +597,61 @@ func (sc *StatementContext) Reset() bool { return true } +// SaveLogicalPlanBuildState captures the statement-scoped planner state before building +// another logical plan candidate from the same AST. +func (sc *StatementContext) SaveLogicalPlanBuildState() LogicalPlanBuildState { + planCacheUseCache, planCacheType, planCacheUnqualified, planCacheForce, planCacheAlwaysWarn := sc.PlanCacheTracker.Save() + return LogicalPlanBuildState{ + warnings: slices.Clone(sc.GetWarnings()), + extraWarnings: slices.Clone(sc.GetExtraWarnings()), + tables: slices.Clone(sc.Tables), + tableStats: maps.Clone(sc.TableStats), + lockTableIDs: maps.Clone(sc.LockTableIDs), + tblInfo2UnionScan: maps.Clone(sc.TblInfo2UnionScan), + useDynamicPruneMode: sc.UseDynamicPruneMode, + colRefFromUpdatePlan: sc.ColRefFromUpdatePlan.Copy(), + planCacheUseCache: planCacheUseCache, + planCacheType: planCacheType, + planCacheUnqualified: planCacheUnqualified, + planCacheForce: planCacheForce, + planCacheAlwaysWarn: planCacheAlwaysWarn, + } +} + +// RestoreLogicalPlanBuildState restores the statement-scoped planner state after a +// discarded logical plan build attempt. +func (sc *StatementContext) RestoreLogicalPlanBuildState(state LogicalPlanBuildState) { + sc.SetWarnings(slices.Clone(state.warnings)) + sc.SetExtraWarnings(slices.Clone(state.extraWarnings)) + sc.Tables = slices.Clone(state.tables) + sc.TableStats = maps.Clone(state.tableStats) + sc.LockTableIDs = maps.Clone(state.lockTableIDs) + sc.TblInfo2UnionScan = maps.Clone(state.tblInfo2UnionScan) + sc.UseDynamicPruneMode = state.useDynamicPruneMode + sc.ColRefFromUpdatePlan.CopyFrom(state.colRefFromUpdatePlan) + sc.PlanCacheTracker.Restore(state.planCacheUseCache, state.planCacheType, state.planCacheUnqualified, state.planCacheForce, state.planCacheAlwaysWarn) + sc.RangeFallbackHandler = contextutil.NewRangeFallbackHandler(&sc.PlanCacheTracker, sc) +} + +// ResetAlternativeLogicalPlanSignals clears the statement-local signals used by the +// alternative logical plan feature. +func (sc *StatementContext) ResetAlternativeLogicalPlanSignals() { + sc.AlternativeLogicalPlanDecorrelatedApply = false + sc.AlternativeLogicalPlanSameOrderIndexJoin = false +} + +// MarkAlternativeLogicalPlanDecorrelatedApply records that at least one Apply has +// been decorrelated into a Join in the current round. +func (sc *StatementContext) MarkAlternativeLogicalPlanDecorrelatedApply() { + sc.AlternativeLogicalPlanDecorrelatedApply = true +} + +// MarkAlternativeLogicalPlanSameOrderIndexJoin records that the current first round +// has already produced a same-order index join candidate for a decorrelated Apply. +func (sc *StatementContext) MarkAlternativeLogicalPlanSameOrderIndexJoin() { + sc.AlternativeLogicalPlanSameOrderIndexJoin = true +} + // CtxID returns the context id of the statement func (sc *StatementContext) CtxID() uint64 { return sc.ctxID diff --git a/pkg/sessionctx/stmtctx/stmtctx_test.go b/pkg/sessionctx/stmtctx/stmtctx_test.go index b5d61cfe5a036..2eeed9ce65bb5 100644 --- a/pkg/sessionctx/stmtctx/stmtctx_test.go +++ b/pkg/sessionctx/stmtctx/stmtctx_test.go @@ -27,6 +27,9 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tidb/pkg/errctx" "github.com/pingcap/tidb/pkg/kv" + "github.com/pingcap/tidb/pkg/meta/model" + "github.com/pingcap/tidb/pkg/parser/ast" + pmodel "github.com/pingcap/tidb/pkg/parser/model" "github.com/pingcap/tidb/pkg/sessionctx/stmtctx" "github.com/pingcap/tidb/pkg/sessionctx/variable" "github.com/pingcap/tidb/pkg/testkit" @@ -218,6 +221,87 @@ func TestMarshalSQLWarn(t *testing.T) { tk.MustQuery("show warnings").Check(rows) } +func TestLogicalPlanBuildStateRestore(t *testing.T) { + sc := stmtctx.NewStmtCtx() + sc.AppendWarning(errors.New("baseline warning")) + sc.AppendExtraWarning(errors.New("baseline extra warning")) + sc.Tables = []stmtctx.TableEntry{{DB: "test", Table: "t"}} + sc.TableStats = map[int64]any{42: "baseline stats"} + sc.LockTableIDs = map[int64]struct{}{1: {}} + tblInfo := &model.TableInfo{ID: 42} + sc.TblInfo2UnionScan = map[*model.TableInfo]bool{tblInfo: true} + sc.UseDynamicPruneMode = true + sc.ColRefFromUpdatePlan.Insert(7) + sc.SetCacheType(contextutil.SessionNonPrepared) + sc.EnablePlanCache() + + state := sc.SaveLogicalPlanBuildState() + + sc.AppendWarning(errors.New("candidate warning")) + sc.AppendExtraWarning(errors.New("candidate extra warning")) + sc.Tables = []stmtctx.TableEntry{{DB: "candidate", Table: "t2"}} + sc.TableStats = map[int64]any{99: "candidate stats"} + sc.LockTableIDs[2] = struct{}{} + sc.TblInfo2UnionScan = map[*model.TableInfo]bool{{ID: 99}: false} + sc.UseDynamicPruneMode = false + sc.ColRefFromUpdatePlan.Insert(9) + sc.SetSkipPlanCache("candidate reason") + + sc.RestoreLogicalPlanBuildState(state) + + warnings := sc.GetWarnings() + require.Len(t, warnings, 1) + require.Equal(t, "baseline warning", warnings[0].Err.Error()) + + extraWarnings := sc.GetExtraWarnings() + require.Len(t, extraWarnings, 1) + require.Equal(t, "baseline extra warning", extraWarnings[0].Err.Error()) + + require.Equal(t, []stmtctx.TableEntry{{DB: "test", Table: "t"}}, sc.Tables) + require.Equal(t, map[int64]any{42: "baseline stats"}, sc.TableStats) + require.Equal(t, map[int64]struct{}{1: {}}, sc.LockTableIDs) + require.Equal(t, map[*model.TableInfo]bool{tblInfo: true}, sc.TblInfo2UnionScan) + require.True(t, sc.UseDynamicPartitionPrune()) + require.True(t, sc.ColRefFromUpdatePlan.Has(7)) + require.False(t, sc.ColRefFromUpdatePlan.Has(9)) + require.True(t, sc.UseCache()) + require.Empty(t, sc.PlanCacheUnqualified()) +} + +func TestQBHintHandlerBuildState(t *testing.T) { + handler := hint.NewQBHintHandler(nil) + handler.QBNameToSelOffset = map[string]int{"qb_1": 1} + handler.ViewQBNameToTable = map[string][]ast.HintTable{ + "view_qb": {{TableName: pmodel.NewCIStr("t")}}, + } + handler.ViewQBNameToHints = map[string][]*ast.TableOptimizerHint{ + "view_qb": {{HintName: pmodel.NewCIStr("merge_join")}}, + } + handler.Enter(&ast.SelectStmt{}) + handler.Enter(&ast.SelectStmt{}) + state := handler.NewBuildState() + hints := handler.GetCurrentStmtHints([]*ast.TableOptimizerHint{ + {HintName: pmodel.NewCIStr("use_index"), QBName: pmodel.NewCIStr("qb_1")}, + }, 1, state) + handler.MarkViewQBNameUsed("view_qb", state) + + require.Len(t, hints, 1) + require.Equal(t, "use_index", hints[0].HintName.L) + + require.Equal(t, 2, handler.MaxSelectStmtOffset()) + require.Equal(t, map[string]int{"qb_1": 1}, handler.QBNameToSelOffset) + require.Equal(t, map[string][]*ast.TableOptimizerHint{ + "view_qb": {{HintName: pmodel.NewCIStr("merge_join")}}, + }, handler.ViewQBNameToHints) + require.Equal(t, map[string][]ast.HintTable{ + "view_qb": {{TableName: pmodel.NewCIStr("t")}}, + }, handler.ViewQBNameToTable) + require.Equal(t, map[int][]*ast.TableOptimizerHint{ + 1: {{HintName: pmodel.NewCIStr("use_index"), QBName: pmodel.NewCIStr("qb_1")}}, + }, state.QBOffsetToHints) + require.Equal(t, map[string]struct{}{"view_qb": {}}, state.ViewQBNameUsed) +} + func TestApproxRuntimeInfo(t *testing.T) { var n = rand.Intn(19000) + 1000 var valRange = rand.Int31n(10000) + 1000 diff --git a/pkg/sessionctx/variable/session.go b/pkg/sessionctx/variable/session.go index 94ed72e817207..c4da8666acb33 100644 --- a/pkg/sessionctx/variable/session.go +++ b/pkg/sessionctx/variable/session.go @@ -1124,6 +1124,11 @@ type SessionVars struct { // EnableNoDecorrelateInSelect enables the NO_DECORRELATE hint for subqueries in the select list. EnableNoDecorrelateInSelect bool + // EnableAlternativeLogicalPlans enables building an extra non-decorrelate + // logical alternative when decorrelation does not produce an equivalent + // same-order index join candidate. + EnableAlternativeLogicalPlans bool + // EnableSemiJoinRewrite enables the SEMI_JOIN_REWRITE hint for subqueries in the where clause. EnableSemiJoinRewrite bool @@ -2184,6 +2189,7 @@ func NewSessionVars(hctx HookContext) *SessionVars { OptimizerSelectivityLevel: DefTiDBOptimizerSelectivityLevel, EnableOuterJoinReorder: DefTiDBEnableOuterJoinReorder, EnableNoDecorrelateInSelect: DefOptEnableNoDecorrelateInSelect, + EnableAlternativeLogicalPlans: DefOptEnableAlternativeLogicalPlans, RetryLimit: DefTiDBRetryLimit, DisableTxnAutoRetry: DefTiDBDisableTxnAutoRetry, DDLReorgPriority: kv.PriorityLow, diff --git a/pkg/sessionctx/variable/setvar_affect.go b/pkg/sessionctx/variable/setvar_affect.go index d581c53643da1..9752fda21d755 100644 --- a/pkg/sessionctx/variable/setvar_affect.go +++ b/pkg/sessionctx/variable/setvar_affect.go @@ -112,6 +112,7 @@ var isHintUpdatableVerified = map[string]struct{}{ "mpp_version": {}, "tidb_enable_inl_join_inner_multi_pattern": {}, "tidb_opt_enable_no_decorrelate_in_select": {}, + "tidb_opt_enable_alternative_logical_plans": {}, "tidb_opt_enable_late_materialization": {}, "tidb_opt_ordering_index_selectivity_threshold": {}, "tidb_opt_ordering_index_selectivity_ratio": {}, diff --git a/pkg/sessionctx/variable/sysvar.go b/pkg/sessionctx/variable/sysvar.go index d2dad495efdeb..bdc2106f7f6a6 100644 --- a/pkg/sessionctx/variable/sysvar.go +++ b/pkg/sessionctx/variable/sysvar.go @@ -2343,6 +2343,10 @@ var defaultSysVars = []*SysVar{ s.EnableNoDecorrelateInSelect = TiDBOptOn(val) return nil }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptEnableAlternativeLogicalPlans, Value: BoolToOnOff(DefOptEnableAlternativeLogicalPlans), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + s.EnableAlternativeLogicalPlans = TiDBOptOn(val) + return nil + }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableStrictDoubleTypeCheck, Value: BoolToOnOff(DefEnableStrictDoubleTypeCheck), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { s.EnableStrictDoubleTypeCheck = TiDBOptOn(val) return nil diff --git a/pkg/sessionctx/variable/tidb_vars.go b/pkg/sessionctx/variable/tidb_vars.go index e929a9dad0827..ac2c865175b4c 100644 --- a/pkg/sessionctx/variable/tidb_vars.go +++ b/pkg/sessionctx/variable/tidb_vars.go @@ -333,6 +333,11 @@ const ( // TiDBOptEnableNoDecorrelateInSelect is the variable equivalent of NO_DECORRELATE hint. TiDBOptEnableNoDecorrelateInSelect = "tidb_opt_enable_no_decorrelate_in_select" + // TiDBOptEnableAlternativeLogicalPlans controls whether the optimizer may build + // an extra non-decorrelate logical alternative when decorrelation does not + // produce an equivalent same-order index join candidate. + TiDBOptEnableAlternativeLogicalPlans = "tidb_opt_enable_alternative_logical_plans" + // TiDBOptLimitPushDownThreshold determines if push Limit or TopN down to TiKV forcibly. TiDBOptLimitPushDownThreshold = "tidb_opt_limit_push_down_threshold" @@ -1322,6 +1327,7 @@ const ( DefOptWriteRowID = false DefOptEnableCorrelationAdjustment = true DefOptEnableNoDecorrelateInSelect = false + DefOptEnableAlternativeLogicalPlans = false DefOptEnableSemiJoinRewrite = false DefOptLimitPushDownThreshold = 100 DefOptCorrelationThreshold = 0.9 diff --git a/pkg/sessionctx/variable/varsutil_test.go b/pkg/sessionctx/variable/varsutil_test.go index f72b090298ff7..19b2af9aeb43a 100644 --- a/pkg/sessionctx/variable/varsutil_test.go +++ b/pkg/sessionctx/variable/varsutil_test.go @@ -92,6 +92,7 @@ func TestNewSessionVars(t *testing.T) { require.Equal(t, DefTiDBAnalyzeVersion, vars.AnalyzeVersion) require.Equal(t, DefCTEMaxRecursionDepth, vars.CTEMaxRecursionDepth) require.Equal(t, int64(DefTiDBTmpTableMaxSize), vars.TMPTableSize) + require.Equal(t, DefOptEnableAlternativeLogicalPlans, vars.EnableAlternativeLogicalPlans) assertFieldsGreaterThanZero(t, reflect.ValueOf(vars.MemQuota)) assertFieldsGreaterThanZero(t, reflect.ValueOf(vars.BatchSize)) @@ -217,6 +218,11 @@ func TestVarsutil(t *testing.T) { require.NoError(t, err) require.True(t, v.BatchInsert) + require.False(t, v.EnableAlternativeLogicalPlans) + err = v.SetSystemVar(TiDBOptEnableAlternativeLogicalPlans, "1") + require.NoError(t, err) + require.True(t, v.EnableAlternativeLogicalPlans) + require.Equal(t, 32, v.InitChunkSize) require.Equal(t, 1024, v.MaxChunkSize) err = v.SetSystemVar(TiDBMaxChunkSize, "2") diff --git a/pkg/statistics/handle/handletest/BUILD.bazel b/pkg/statistics/handle/handletest/BUILD.bazel index 3c420016e834f..6fad30bf0a37d 100644 --- a/pkg/statistics/handle/handletest/BUILD.bazel +++ b/pkg/statistics/handle/handletest/BUILD.bazel @@ -9,7 +9,7 @@ go_test( ], flaky = True, race = "on", - shard_count = 37, + shard_count = 38, deps = [ "//pkg/config", "//pkg/domain", diff --git a/pkg/util/context/plancache.go b/pkg/util/context/plancache.go index 9cac5ec81012e..d6c46cd4db81e 100644 --- a/pkg/util/context/plancache.go +++ b/pkg/util/context/plancache.go @@ -122,6 +122,26 @@ func (h *PlanCacheTracker) EnablePlanCache() { h.useCache = true } +// Save captures the mutable planning-time state of the tracker. +func (h *PlanCacheTracker) Save() (useCache bool, cacheType PlanCacheType, planCacheUnqualified string, forcePlanCache bool, alwaysWarnSkipCache bool) { + h.mu.Lock() + defer h.mu.Unlock() + + return h.useCache, h.cacheType, h.planCacheUnqualified, h.forcePlanCache, h.alwaysWarnSkipCache +} + +// Restore restores the mutable planning-time state of the tracker. +func (h *PlanCacheTracker) Restore(useCache bool, cacheType PlanCacheType, planCacheUnqualified string, forcePlanCache bool, alwaysWarnSkipCache bool) { + h.mu.Lock() + defer h.mu.Unlock() + + h.useCache = useCache + h.cacheType = cacheType + h.planCacheUnqualified = planCacheUnqualified + h.forcePlanCache = forcePlanCache + h.alwaysWarnSkipCache = alwaysWarnSkipCache +} + // UseCache returns whether to use plan cache. func (h *PlanCacheTracker) UseCache() bool { h.mu.Lock() diff --git a/pkg/util/hint/hint_query_block.go b/pkg/util/hint/hint_query_block.go index 99bfc646837e6..3b98e93a69bd0 100644 --- a/pkg/util/hint/hint_query_block.go +++ b/pkg/util/hint/hint_query_block.go @@ -31,18 +31,23 @@ import ( // In both cases, the `use_index` hint doesn't take effect directly, since a specific qb_name is specified, and this // QBHintHandler is used to handle this cases. type QBHintHandler struct { - QBNameToSelOffset map[string]int // map[QBName]SelectOffset - QBOffsetToHints map[int][]*ast.TableOptimizerHint // map[QueryBlockOffset]Hints + QBNameToSelOffset map[string]int // map[QBName]SelectOffset // Used for the view's hint ViewQBNameToTable map[string][]ast.HintTable // map[QBName]HintedTable ViewQBNameToHints map[string][]*ast.TableOptimizerHint // map[QBName]Hints - ViewQBNameUsed map[string]struct{} // map[QBName]Used warnHandler hintWarnHandler selectStmtOffset int } +// QBHintBuildState stores the per-build runtime state for a QBHintHandler. +// The handler itself only keeps AST-derived metadata and can be shared safely. +type QBHintBuildState struct { + QBOffsetToHints map[int][]*ast.TableOptimizerHint // map[QueryBlockOffset]Hints + ViewQBNameUsed map[string]struct{} // map[QBName]Used +} + // hintWarnHandler is used to handle the warning when parsing hints. type hintWarnHandler interface { SetHintWarning(warn string) @@ -56,6 +61,20 @@ func NewQBHintHandler(warnHandler hintWarnHandler) *QBHintHandler { } } +// NewBuildState creates the per-build runtime state for the handler. +func (p *QBHintHandler) NewBuildState() *QBHintBuildState { + if p == nil { + return nil + } + + state := &QBHintBuildState{} + if len(p.ViewQBNameToTable) == 0 { + return state + } + state.ViewQBNameUsed = make(map[string]struct{}, len(p.ViewQBNameToTable)) + return state +} + // MaxSelectStmtOffset returns the current stmt offset. func (p *QBHintHandler) MaxSelectStmtOffset() int { return p.selectStmtOffset @@ -133,7 +152,6 @@ func (p *QBHintHandler) handleViewHints(hints []*ast.TableOptimizerHint, offset usedHints[i] = true if p.ViewQBNameToTable == nil { p.ViewQBNameToTable = make(map[string][]ast.HintTable) - p.ViewQBNameUsed = make(map[string]struct{}) } qbName := hint.QBName.L if qbName == "" { @@ -202,15 +220,21 @@ func (p *QBHintHandler) handleViewHints(hints []*ast.TableOptimizerHint, offset } // HandleUnusedViewHints handle the unused view hints. -func (p *QBHintHandler) HandleUnusedViewHints() { +func (p *QBHintHandler) HandleUnusedViewHints(state *QBHintBuildState, warn []string) []string { + if state == nil { + return warn + } + + warn = warn[:0] if p.ViewQBNameToTable != nil { for qbName := range p.ViewQBNameToTable { - _, ok := p.ViewQBNameUsed[qbName] + _, ok := state.ViewQBNameUsed[qbName] if !ok && p.warnHandler != nil { - p.warnHandler.SetHintWarning(fmt.Sprintf("The qb_name hint %s is unused, please check whether the table list in the qb_name hint %s is correct", qbName, qbName)) + warn = append(warn, fmt.Sprintf("The qb_name hint %s is unused, please check whether the table list in the qb_name hint %s is correct", qbName, qbName)) } } } + return warn } const ( @@ -243,6 +267,16 @@ func (p *QBHintHandler) getBlockOffset(blockName model.CIStr) int { return -1 } +// SetWarns set the warning from a list of strings. +func (p *QBHintHandler) SetWarns(warns []string) { + if p == nil || p.warnHandler == nil || len(warns) == 0 { + return + } + for _, one := range warns { + p.warnHandler.SetHintWarning(one) + } +} + // GetHintOffset gets the offset of stmt that the hints take effects. func (p *QBHintHandler) GetHintOffset(qbName model.CIStr, currentOffset int) int { if qbName.L != "" { @@ -280,9 +314,12 @@ func (p *QBHintHandler) isHint4View(hint *ast.TableOptimizerHint) bool { } // GetCurrentStmtHints extracts all hints that take effects at current stmt. -func (p *QBHintHandler) GetCurrentStmtHints(hints []*ast.TableOptimizerHint, currentOffset int) []*ast.TableOptimizerHint { - if p.QBOffsetToHints == nil { - p.QBOffsetToHints = make(map[int][]*ast.TableOptimizerHint) +func (p *QBHintHandler) GetCurrentStmtHints(hints []*ast.TableOptimizerHint, currentOffset int, state *QBHintBuildState) []*ast.TableOptimizerHint { + if state == nil { + state = &QBHintBuildState{} + } + if state.QBOffsetToHints == nil { + state.QBOffsetToHints = make(map[int][]*ast.TableOptimizerHint) } for _, hint := range hints { if hint.HintName.L == hintQBName { @@ -296,11 +333,19 @@ func (p *QBHintHandler) GetCurrentStmtHints(hints []*ast.TableOptimizerHint, cur } continue } - if !slices.Contains(p.QBOffsetToHints[offset], hint) { - p.QBOffsetToHints[offset] = append(p.QBOffsetToHints[offset], hint) + if !slices.Contains(state.QBOffsetToHints[offset], hint) { + state.QBOffsetToHints[offset] = append(state.QBOffsetToHints[offset], hint) } } - return p.QBOffsetToHints[currentOffset] + return state.QBOffsetToHints[currentOffset] +} + +// MarkViewQBNameUsed records that the named view qb hint was used in the current build. +func (*QBHintHandler) MarkViewQBNameUsed(qbName string, state *QBHintBuildState) { + if state == nil || state.ViewQBNameUsed == nil { + return + } + state.ViewQBNameUsed[qbName] = struct{}{} } // GenerateQBName builds QBName from offset.