pkg/planner, pkg/sessionctx: keep native TiCI FTS plan when LIKE fallback rejects syntax | tidb-test=feature/fts tiflash=feature/fts tikv=feature/fts

docs/note/planner/rule/rule_ai_notes.md

@@ -45,3 +45,20 @@ Test and verification:
 - Add SQL-only case to `predicate_pushdown_suite_in.json`; keep DDL in the test setup, otherwise `explain` will try to run `DROP/CREATE`.
 - Record with: `go test ./pkg/planner/core/casetest/rule -run TestConstantPropagateWithCollation --tags=intest -record`.
 - Add integration test to `tests/integrationtest/t/select.test` and record via `./run-tests.sh -r select` (integration tests use `-r`, not `-record`).
+
+## 2026-05-19 - FTS alt-plan heuristic invalidated native TiCI plans
+
+Background:
+- `MATCH ... AGAINST` in direct-boolean predicate context can run through the alternative logical-plan framework, where round 1 keeps the native TiCI/TiFlash path and a later round may try the LIKE fallback.
+- A planner-side heuristic (`ftsNativeViable`) was used during expression rewrite to pre-mark round 1 as non-viable before the real TiCI planning path ran.
+- That heuristic drifted from the native implementation and could reject executable native TiCI plans, causing round 1 to be discarded and unsupported LIKE fallback errors (for example BOOLEAN prefix queries like `stainles*`) to leak to users.
+
+Implementation choice:
+- Remove the `nonViableFTSMatch` build-time invalidation signal.
+- Keep `HasPredicateMatch` only for cost competition / fallback-round eligibility.
+- Let the real native planning path (`DoOptimize` / TiCI analysis) decide when fallback should take over via `FTSLikeFallbackError`.
+
+Test and verification:
+- Add a TiCI regression test covering `@@tidb_opt_enable_alternative_logical_plans = 1` with a native multi-column prefix query that must keep the TiCI plan.
+- Run with failpoints enabled:
+  `make failpoint-enable && (go test ./pkg/planner/core/casetest/tici -run TestTiCIAlternativeLogicalPlansKeepNativePrefixPlan --tags=intest; rc=$?; make failpoint-disable; exit $rc)`

pkg/planner/core/casetest/tici/BUILD.bazel

@@ -10,12 +10,13 @@ go_test(
     ],
     data = glob(["testdata/**"]),
     flaky = True,
-    shard_count = 12,
+    shard_count = 15,
     deps = [
         "//pkg/config",
         "//pkg/ddl/ingest/testutil",
         "//pkg/domain",
         "//pkg/domain/infosync",
+        "//pkg/sessionctx/vardef",
         "//pkg/store/mockstore",
         "//pkg/testkit",
         "//pkg/testkit/testdata",

pkg/planner/core/casetest/tici/stats_test.go

@@ -24,6 +24,7 @@ import (
 	ingesttestutil "github.com/pingcap/tidb/pkg/ddl/ingest/testutil"
 	"github.com/pingcap/tidb/pkg/domain"
 	"github.com/pingcap/tidb/pkg/domain/infosync"
+	"github.com/pingcap/tidb/pkg/sessionctx/vardef"
 	"github.com/pingcap/tidb/pkg/store/mockstore"
 	"github.com/pingcap/tidb/pkg/testkit"
 	"github.com/pingcap/tidb/pkg/testkit/testdata"
@@ -43,6 +44,8 @@ func TestTiCISearchEstimateOnlyForMultiTable(t *testing.T) {
 	store := testkit.CreateMockStoreWithSchemaLease(t, time.Second, mockstore.WithMockTiFlash(2))
 	defer ingesttestutil.InjectMockBackendCtx(t, store)()
 	tk := testkit.NewTestKit(t, store)
+	tk.MustExec(fmt.Sprintf("set global %s = on", vardef.TiDBEnableTiCIEstimate))
+	defer tk.MustExec(fmt.Sprintf("set global %s = on", vardef.TiDBEnableTiCIEstimate))
 
 	tiflash := infosync.NewMockTiFlash()
 	infosync.SetMockTiFlash(tiflash)
@@ -75,6 +78,11 @@ func TestTiCISearchEstimateOnlyForMultiTable(t *testing.T) {
 	multiTablePlan := testdata.ConvertRowsToStrings(
 		tk.MustQuery("explain format='brief' select /*+ inl_join(t) */ t.id from t2, t where t.id = t2.a and fts_match_word('hello', t.title)").Rows())
 	requirePlanLineContains(t, multiTablePlan, "IndexRangeScan 100.00", "search func:fts_match_word")
+
+	tk.MustExec(fmt.Sprintf("set global %s = off", vardef.TiDBEnableTiCIEstimate))
+	disabledEstimatePlan := testdata.ConvertRowsToStrings(
+		tk.MustQuery("explain format='brief' select /*+ inl_join(t) */ t.id from t2, t where t.id = t2.a and match(t.title) against ('hello' in boolean mode)").Rows())
+	requirePlanLineContains(t, disabledEstimatePlan, "IndexRangeScan 10.00", "search func:fts_match_word")
 }
 
 func requirePlanLineContains(t *testing.T, plan []string, expected ...string) {

pkg/planner/core/casetest/tici/tici_test.go

@@ -580,3 +580,85 @@ func TestTiCIJoinWithNonTiCITable(t *testing.T) {
 		tk.MustQuery(sql).CheckNotContain("mpp[tiflash]")
 	})
 }
+
+func TestTiCIAlternativeLogicalPlansKeepNativePrefixPlan(t *testing.T) {
+	runTiCITest(t, func(tk *testkit.TestKit) {
+		tk.MustExec(`create table amazon_review(
+			id bigint primary key,
+			review_body text,
+			review_headline text,
+			product_title text,
+			fulltext index review_body(review_body, review_headline, product_title)
+		)`)
+		dom := domain.GetDomain(tk.Session())
+		testkit.SetTiFlashReplica(t, dom, "test", "amazon_review")
+		tk.MustExec("set @@tidb_opt_enable_alternative_logical_plans = 1")
+
+		sql := `explain format='brief' select count(*) from amazon_review
+			where match(review_body, review_headline, product_title)
+			against('stainles*' in boolean mode)`
+		tk.MustQuery(sql).CheckContain(`search func:fts_match_prefix("stainles", test.amazon_review.review_body, test.amazon_review.review_headline, test.amazon_review.product_title)`)
+		tk.MustQuery(sql).CheckContain("index:review_body(review_body, review_headline, product_title)")
+	})
+}
+
+func TestTiCIAlternativeLogicalPlansPreferNativeOverLikeFallbackOnCost(t *testing.T) {
+	runTiCITest(t, func(tk *testkit.TestKit) {
+		tk.MustExec(`create table amazon_review(
+			id bigint primary key,
+			review_body text,
+			review_headline text,
+			product_title text,
+			fulltext index review_body(review_body, review_headline, product_title)
+		)`)
+		tk.MustExec(`insert into amazon_review values
+			(1, 'stainless steel bottle', 'durable bottle', 'travel bottle'),
+			(2, 'stainless lunch box', 'steel lunch box', 'kitchen organizer'),
+			(3, 'plastic container', 'light container', 'storage box'),
+			(4, 'ceramic mug', 'coffee mug', 'kitchen mug')`)
+		dom := domain.GetDomain(tk.Session())
+		testkit.SetTiFlashReplica(t, dom, "test", "amazon_review")
+
+		tk.MustExec("set @@tidb_opt_enable_alternative_logical_plans = 1")
+		sql := `explain format='brief' select count(*) from amazon_review
+			where match(review_body, review_headline, product_title)
+			against('stainless' in boolean mode)`
+		tk.MustQuery(sql).CheckContain(`search func:fts_match_word("stainless", test.amazon_review.review_body, test.amazon_review.review_headline, test.amazon_review.product_title)`)
+		tk.MustQuery(sql).CheckContain("index:review_body(review_body, review_headline, product_title)")
+		tk.MustQuery(sql).CheckNotContain("ilike")
+	})
+}
+
+func TestTiCIAlternativeLogicalPlansIgnoreIndexRoutesToLikeFallback(t *testing.T) {
+	runTiCITest(t, func(tk *testkit.TestKit) {
+		tk.MustExec(`create table amazon_review(
+			id bigint primary key,
+			review_body text,
+			review_headline text,
+			product_title text,
+			fulltext index review_body(review_body, review_headline, product_title)
+		)`)
+		tk.MustExec(`insert into amazon_review values
+			(1, 'stainless steel bottle', 'durable bottle', 'travel bottle'),
+			(2, 'stainless lunch box', 'steel lunch box', 'kitchen organizer'),
+			(3, 'plastic container', 'light container', 'storage box'),
+			(4, 'ceramic mug', 'coffee mug', 'kitchen mug')`)
+		dom := domain.GetDomain(tk.Session())
+		testkit.SetTiFlashReplica(t, dom, "test", "amazon_review")
+		tk.MustExec("set @@tidb_opt_enable_alternative_logical_plans = 1")
+
+		sqlWithIgnoreIndex := `explain format='brief' select count(*) from amazon_review ignore index(review_body)
+			where match(review_body, review_headline, product_title)
+			against('stainless' in boolean mode)`
+		tk.MustQuery(sqlWithIgnoreIndex).CheckContain("ilike(")
+		tk.MustQuery(sqlWithIgnoreIndex).CheckContain("TableFullScan")
+		tk.MustQuery(sqlWithIgnoreIndex).CheckNotContain("fts_match_word")
+
+		sqlWithIgnoreHint := `explain format='brief' select /*+ ignore_index(ar, review_body) */ count(*) from amazon_review ar
+			where match(review_body, review_headline, product_title)
+			against('stainless' in boolean mode)`
+		tk.MustQuery(sqlWithIgnoreHint).CheckContain("ilike(")
+		tk.MustQuery(sqlWithIgnoreHint).CheckContain("TableFullScan")
+		tk.MustQuery(sqlWithIgnoreHint).CheckNotContain("fts_match_word")
+	})
+}

pkg/planner/core/expression_rewriter.go

@@ -730,9 +730,9 @@ func (er *expressionRewriter) inDirectMatchBooleanContext() bool {
 // plan and rebuild via the fts-like-fallback round. It is used by the modifier
 // guard in matchAgainstToBuiltin to allow native emission of a non-default
 // modifier when round 1's plan is destined for discard anyway. The rescue
-// conditions mirror the ones in matchAgainstToExpression that trigger
-// MarkNonViableFTSMatch — alternative logical plans enabled AND a direct
-// boolean predicate context.
+// conditions mirror the ones in matchAgainstToExpression that trigger the
+// fallback round — alternative logical plans enabled AND a direct boolean
+// predicate context.
 func (er *expressionRewriter) matchHasLikeFallbackRescue() bool {
 	if er.planCtx == nil || er.planCtx.builder == nil || er.planCtx.builder.ctx == nil {
 		return false
@@ -2368,13 +2368,11 @@ func (er *expressionRewriter) matchAgainstToExpression(v *ast.MatchAgainst) {
 	// etc. would silently produce wrong rows if the LIKE rewrite's integer
 	// result were substituted for the native float score.
 	//
-	// Round 1 also has to record viability before committing to native: if
-	// any boolean-context MATCH is non-viable, the resulting plan would
-	// fail at execution. The rewriter records that on the planBuilder so the
-	// round driver can invalidate the plan and trigger the fallback round.
-	// Round 1 additionally records that a direct-boolean-context MATCH was
-	// seen so the driver runs the LIKE round for cost competition even when
-	// round 1's native plan is executable.
+	// Round 1 records that a direct-boolean-context MATCH was seen so the
+	// round driver runs the LIKE round for cost competition when alternative
+	// logical plans are enabled. Whether the native plan is actually usable is
+	// decided by the real native planning path (DoOptimize / analyzeTiCIIndex),
+	// not by a separate heuristic here.
 	useLikeFallback := false
 	if er.planCtx != nil && er.planCtx.builder != nil && er.planCtx.builder.ctx != nil {
 		sessVars := er.planCtx.builder.ctx.GetSessionVars()
@@ -2384,14 +2382,10 @@ func (er *expressionRewriter) matchAgainstToExpression(v *ast.MatchAgainst) {
 				useLikeFallback = true
 			} else if sessVars.EnableAlternativeLogicalPlans {
 				// Round 1 (native). Mark the build so the driver runs the LIKE
-				// round and cost-compares its plan against round 1's. If this
-				// MATCH cannot run natively, also mark the build as non-viable
-				// so the driver discards round 1's plan; the rewrite continues
-				// with the native builtin to keep round 1 internally consistent.
+				// round and cost-compares its plan against round 1's. If native
+				// planning later fails, optimize.go will arm the fallback round
+				// from the real native error path.
 				er.planCtx.builder.MarkPredicateMatch()
-				if !er.ftsNativeViable(v.Modifier, numCols, stackLen) {
-					er.planCtx.builder.MarkNonViableFTSMatch()
-				}
 			}
 		}
 	}

pkg/planner/core/operator/logicalop/logical_datasource.go

@@ -734,6 +734,9 @@ func (ds *DataSource) chooseTiCIIndex(
 	matchedIndexIsHinted := false
 
 	for _, path := range ds.AllPossibleAccessPaths {
+		if ds.isTiCIIndexIgnoredByHint(path.Index) {
+			continue
+		}
 		if !ds.isTiCIIndexPathCandidate(path, hasFTSFuncLocal, matchedIndexIsHinted) {
 			continue
 		}
@@ -757,6 +760,42 @@ func (ds *DataSource) chooseTiCIIndex(
 	return matchedIdx, matchedExprSetForChosenIndex, hasUnmatchedFTSOverAllIdx
 }
 
+func (ds *DataSource) isTiCIIndexIgnoredByHint(index *model.IndexInfo) bool {
+	if index == nil || !index.IsTiCIIndex() {
+		return false
+	}
+	for _, hint := range ds.AstIndexHints {
+		if isIgnoredByIndexHint(index, hint) {
+			return true
+		}
+	}
+	tblName := ds.TableInfo.Name
+	if ds.TableAsName != nil && ds.TableAsName.L != "" {
+		tblName = *ds.TableAsName
+	}
+	for _, hintedIdx := range ds.IndexHints {
+		if !hintedIdx.Match(ds.DBName, tblName) {
+			continue
+		}
+		if isIgnoredByIndexHint(index, hintedIdx.IndexHint) {
+			return true
+		}
+	}
+	return false
+}
+
+func isIgnoredByIndexHint(index *model.IndexInfo, hint *ast.IndexHint) bool {
+	if hint == nil || hint.HintScope != ast.HintForScan || hint.HintType != ast.HintIgnore {
+		return false
+	}
+	for _, idxName := range hint.IndexNames {
+		if idxName.L == index.Name.L {
+			return true
+		}
+	}
+	return false
+}
+
 func (ds *DataSource) isTiCIIndexPathCandidate(path *util.AccessPath, hasFTSFuncLocal bool, matchedIndexIsHinted bool) bool {
 	// Not TiCI index, skip it.
 	if path.Index == nil || !path.Index.IsTiCIIndex() {

pkg/planner/core/planbuilder.go

@@ -316,15 +316,6 @@ type PlanBuilder struct {
 	// resolveCtx is set when calling Build, it's only effective in the current Build call.
 	resolveCtx *resolve.Context
 
-	// nonViableFTSMatch is set during build when the expression rewriter
-	// encounters a predicate-context MATCH...AGAINST whose native form
-	// (FTSMysqlMatchAgainst) cannot be executed — the matched columns lack a
-	// public FULLTEXT index on a TiFlash-backed table, or the modifier is not
-	// supported by pushdown. The flag is read by the alternative-rounds driver
-	// after the round to invalidate the round's plan and trigger the
-	// fts-like-fallback round (see optimize.go).
-	nonViableFTSMatch bool
-
 	// predicateMatchSeen is set during build when the expression rewriter
 	// encounters a direct-boolean-context MATCH...AGAINST (one whose 0/1 boolean
 	// result is consumed directly as a predicate). The alternative-rounds driver
@@ -333,20 +324,6 @@ type PlanBuilder struct {
 	predicateMatchSeen bool
 }
 
-// HasNonViableFTSMatch reports whether the most recent build round saw a
-// predicate-context MATCH...AGAINST that could not be served by the native
-// FTSMysqlMatchAgainst builtin. The caller (optimize.go) uses this to
-// invalidate the round's plan and trigger the fts-like-fallback round.
-func (b *PlanBuilder) HasNonViableFTSMatch() bool {
-	return b.nonViableFTSMatch
-}
-
-// MarkNonViableFTSMatch records that a predicate-context MATCH...AGAINST in
-// the current build cannot be served natively. See HasNonViableFTSMatch.
-func (b *PlanBuilder) MarkNonViableFTSMatch() {
-	b.nonViableFTSMatch = true
-}
-
 // HasPredicateMatch reports whether the most recent build round saw a
 // direct-boolean-context MATCH...AGAINST. The caller (optimize.go) uses this
 // to decide whether to run the fts-like-fallback round for cost competition,

pkg/planner/core/stats.go

@@ -38,6 +38,7 @@ import (
 	"github.com/pingcap/tidb/pkg/planner/util"
 	"github.com/pingcap/tidb/pkg/sessionctx"
 	"github.com/pingcap/tidb/pkg/sessionctx/stmtctx"
+	"github.com/pingcap/tidb/pkg/sessionctx/vardef"
 	"github.com/pingcap/tidb/pkg/statistics"
 	"github.com/pingcap/tidb/pkg/types"
 	h "github.com/pingcap/tidb/pkg/util/hint"
@@ -257,6 +258,9 @@ func deriveTiCISearchPathStats(ds *logicalop.DataSource, path *util.AccessPath)
 	if !ok || path == nil || path.Index == nil || path.FtsQueryInfo == nil || len(path.Ranges) == 0 {
 		return 0, false
 	}
+	if !vardef.EnableTiCIEstimate.Load() {
+		return 0, false
+	}
 	provider, ok := sctx.GetStore().(kv.TiCIEstimateCountProvider)
 	if !ok {
 		return 0, false

pkg/planner/optimize.go

@@ -581,7 +581,7 @@ func buildAndOptimizeLogicalPlanRound(
 		optFlag = optFlagAdjust(optFlag)
 	}
 	// when we are sure about we can have a ILIKE fallback for FTS function, we can
-	// directly ignore the error from AnalyzeTICIIndex phase.
+	// directly ignore the error from AnalyzeTICIIndex phase at first round.
 	finalPlan, cost, err := core.DoOptimize(ctx, sctx, optFlag, logic)
 	if err != nil {
 		if discard, fallbackErr := maybeArmFTSLikeFallback(sctx.GetSessionVars(), builder, err); discard {
@@ -598,17 +598,6 @@ func buildAndOptimizeLogicalPlanRound(
 	if builder.HasPredicateMatch() {
 		sctx.GetSessionVars().StmtCtx.AlternativeLogicalPlanHasPredicateContextMatch = true
 	}
-	// If this round saw a predicate-context MATCH that cannot be served by the
-	// native FTSMysqlMatchAgainst builtin, the produced plan would fail at
-	// execution. Discard it and arm AlternativeLogicalPlanFTSLikeFallback so
-	// any intervening rounds (correlate, etc.) re-rewrite with ILIKE too. The
-	// fts-like-fallback round below also forces this flag during setup; this
-	// outer assignment covers the non-viable case where the flag must stay
-	// true across all subsequent rounds, not just inside the LIKE round.
-	if builder.HasNonViableFTSMatch() {
-		sctx.GetSessionVars().StmtCtx.AlternativeLogicalPlanFTSLikeFallback = true
-		return p, names, false, nil
-	}
 
 	if *bestPlan == nil || cost < *bestCost {
 		*bestCost = cost
@@ -741,17 +730,14 @@ func optimize(ctx context.Context, sctx planctx.PlanContext, node *resolve.NodeW
 		initialLogicalPlanCtx = saveLogicalPlanBuildCtx(sessVars)
 		sessVars.StmtCtx.ResetAlternativeLogicalPlanSignals()
 		// Round 1 always uses the native FTSMysqlMatchAgainst builtin — same as
-		// the Alt-disabled default. The build records two signals on the
+		// the Alt-disabled default. The build records one signal on the
 		// planBuilder when MATCH...AGAINST is seen:
 		//   * HasPredicateMatch: any direct-boolean-context MATCH. The round
 		//     driver propagates this into stmtctx to trigger the
 		//     fts-like-fallback alternative round, which builds a competing
-		//     ILIKE-based plan; the cheaper of the two wins.
-		//   * HasNonViableFTSMatch: a predicate-context MATCH whose native form
-		//     cannot execute (no FTS index / no TiFlash replica / unsupported
-		//     modifier). The round driver discards round 1's plan and forces
-		//     AlternativeLogicalPlanFTSLikeFallback true so all subsequent
-		//     rounds (correlate, etc.) re-rewrite with ILIKE.
+		//     ILIKE-based plan; the cheaper of the two wins. If the real native
+		//     planning path later fails, maybeArmFTSLikeFallback forces the LIKE
+		//     round from the native error.
 	}
 
 	p, names, nonLogical, err := buildAndOptimizeLogicalPlanRound(

pkg/sessionctx/stmtctx/stmtctx.go

@@ -495,12 +495,10 @@ type StatementContext struct {
 	// Round 1 always runs with this flag false. The "fts-like-fallback"
 	// alternative round flips it to true (via its setup/cleanup) while it
 	// builds a competing ILIKE-based plan; the cost-cheapest plan wins via the
-	// normal alt-rounds cost comparison. If round 1's build records a
-	// predicate-context MATCH that cannot be served natively (no FTS index on a
-	// matched column / no TiFlash replica / modifier not pushdown-supported),
-	// optimize.go additionally invalidates round 1's plan and forces this flag
-	// true outside the round so any intervening rounds (correlate, etc.) also
-	// produce executable LIKE-based plans.
+	// normal alt-rounds cost comparison. If round 1's real native planning path
+	// fails with a fallbackable FTS error, optimize.go additionally forces this
+	// flag true outside the round so any intervening rounds (correlate, etc.)
+	// also produce executable LIKE-based plans.
 	AlternativeLogicalPlanFTSLikeFallback bool
 	// AlternativeLogicalPlanHasPredicateContextMatch indicates that round 1
 	// encountered a direct-boolean-context MATCH...AGAINST. The round driver