pkg/planner, pkg/sessionctx: keep native TiCI FTS plan when LIKE fallback rejects syntax | tidb-test=feature/fts tiflash=feature/fts tikv=feature/fts

docs/note/planner/rule/rule_ai_notes.md

@@ -45,3 +45,20 @@ Test and verification:
 - Add SQL-only case to `predicate_pushdown_suite_in.json`; keep DDL in the test setup, otherwise `explain` will try to run `DROP/CREATE`.
 - Record with: `go test ./pkg/planner/core/casetest/rule -run TestConstantPropagateWithCollation --tags=intest -record`.
 - Add integration test to `tests/integrationtest/t/select.test` and record via `./run-tests.sh -r select` (integration tests use `-r`, not `-record`).
+
+## 2026-05-19 - FTS alt-plan heuristic invalidated native TiCI plans
+
+Background:
+- `MATCH ... AGAINST` in direct-boolean predicate context can run through the alternative logical-plan framework, where round 1 keeps the native TiCI/TiFlash path and a later round may try the LIKE fallback.
+- A planner-side heuristic (`ftsNativeViable`) was used during expression rewrite to pre-mark round 1 as non-viable before the real TiCI planning path ran.
+- That heuristic drifted from the native implementation and could reject executable native TiCI plans, causing round 1 to be discarded and unsupported LIKE fallback errors (for example BOOLEAN prefix queries like `stainles*`) to leak to users.
+
+Implementation choice:
+- Remove the `nonViableFTSMatch` build-time invalidation signal.
+- Keep `HasPredicateMatch` only for cost competition / fallback-round eligibility.
+- Let the real native planning path (`DoOptimize` / TiCI analysis) decide when fallback should take over via `FTSLikeFallbackError`.
+
+Test and verification:
+- Add a TiCI regression test covering `@@tidb_opt_enable_alternative_logical_plans = 1` with a native multi-column prefix query that must keep the TiCI plan.
+- Run with failpoints enabled:
+  `make failpoint-enable && (go test ./pkg/planner/core/casetest/tici -run TestTiCIAlternativeLogicalPlansKeepNativePrefixPlan --tags=intest; rc=$?; make failpoint-disable; exit $rc)`

pkg/planner/core/casetest/tici/tici_test.go

@@ -580,3 +580,24 @@ func TestTiCIJoinWithNonTiCITable(t *testing.T) {
 		tk.MustQuery(sql).CheckNotContain("mpp[tiflash]")
 	})
 }
+
+func TestTiCIAlternativeLogicalPlansKeepNativePrefixPlan(t *testing.T) {
+	runTiCITest(t, func(tk *testkit.TestKit) {
+		tk.MustExec(`create table amazon_review(
+			id bigint primary key,
+			review_body text,
+			review_headline text,
+			product_title text,
+			fulltext index review_body(review_body, review_headline, product_title)
+		)`)
+		dom := domain.GetDomain(tk.Session())
+		testkit.SetTiFlashReplica(t, dom, "test", "amazon_review")
+		tk.MustExec("set @@tidb_opt_enable_alternative_logical_plans = 1")
+
+		sql := `explain format='brief' select count(*) from amazon_review
+			where match(review_body, review_headline, product_title)
+			against('stainles*' in boolean mode)`
+		tk.MustQuery(sql).CheckContain(`search func:fts_match_prefix("stainles", test.amazon_review.review_body, test.amazon_review.review_headline, test.amazon_review.product_title)`)
+		tk.MustQuery(sql).CheckContain("index:review_body(review_body, review_headline, product_title)")
+	})
+}

pkg/planner/core/expression_rewriter.go

@@ -730,9 +730,9 @@ func (er *expressionRewriter) inDirectMatchBooleanContext() bool {
 // plan and rebuild via the fts-like-fallback round. It is used by the modifier
 // guard in matchAgainstToBuiltin to allow native emission of a non-default
 // modifier when round 1's plan is destined for discard anyway. The rescue
-// conditions mirror the ones in matchAgainstToExpression that trigger
-// MarkNonViableFTSMatch — alternative logical plans enabled AND a direct
-// boolean predicate context.
+// conditions mirror the ones in matchAgainstToExpression that trigger the
+// fallback round — alternative logical plans enabled AND a direct boolean
+// predicate context.
 func (er *expressionRewriter) matchHasLikeFallbackRescue() bool {
 	if er.planCtx == nil || er.planCtx.builder == nil || er.planCtx.builder.ctx == nil {
 		return false
@@ -2368,13 +2368,11 @@ func (er *expressionRewriter) matchAgainstToExpression(v *ast.MatchAgainst) {
 	// etc. would silently produce wrong rows if the LIKE rewrite's integer
 	// result were substituted for the native float score.
 	//
-	// Round 1 also has to record viability before committing to native: if
-	// any boolean-context MATCH is non-viable, the resulting plan would
-	// fail at execution. The rewriter records that on the planBuilder so the
-	// round driver can invalidate the plan and trigger the fallback round.
-	// Round 1 additionally records that a direct-boolean-context MATCH was
-	// seen so the driver runs the LIKE round for cost competition even when
-	// round 1's native plan is executable.
+	// Round 1 records that a direct-boolean-context MATCH was seen so the
+	// round driver runs the LIKE round for cost competition when alternative
+	// logical plans are enabled. Whether the native plan is actually usable is
+	// decided by the real native planning path (DoOptimize / analyzeTiCIIndex),
+	// not by a separate heuristic here.
 	useLikeFallback := false
 	if er.planCtx != nil && er.planCtx.builder != nil && er.planCtx.builder.ctx != nil {
 		sessVars := er.planCtx.builder.ctx.GetSessionVars()
@@ -2384,14 +2382,10 @@ func (er *expressionRewriter) matchAgainstToExpression(v *ast.MatchAgainst) {
 				useLikeFallback = true
 			} else if sessVars.EnableAlternativeLogicalPlans {
 				// Round 1 (native). Mark the build so the driver runs the LIKE
-				// round and cost-compares its plan against round 1's. If this
-				// MATCH cannot run natively, also mark the build as non-viable
-				// so the driver discards round 1's plan; the rewrite continues
-				// with the native builtin to keep round 1 internally consistent.
+				// round and cost-compares its plan against round 1's. If native
+				// planning later fails, optimize.go will arm the fallback round
+				// from the real native error path.
 				er.planCtx.builder.MarkPredicateMatch()
-				if !er.ftsNativeViable(v.Modifier, numCols, stackLen) {
-					er.planCtx.builder.MarkNonViableFTSMatch()
-				}
 			}
 		}
 	}

pkg/planner/core/planbuilder.go

@@ -316,15 +316,6 @@ type PlanBuilder struct {
 	// resolveCtx is set when calling Build, it's only effective in the current Build call.
 	resolveCtx *resolve.Context
 
-	// nonViableFTSMatch is set during build when the expression rewriter
-	// encounters a predicate-context MATCH...AGAINST whose native form
-	// (FTSMysqlMatchAgainst) cannot be executed — the matched columns lack a
-	// public FULLTEXT index on a TiFlash-backed table, or the modifier is not
-	// supported by pushdown. The flag is read by the alternative-rounds driver
-	// after the round to invalidate the round's plan and trigger the
-	// fts-like-fallback round (see optimize.go).
-	nonViableFTSMatch bool
-
 	// predicateMatchSeen is set during build when the expression rewriter
 	// encounters a direct-boolean-context MATCH...AGAINST (one whose 0/1 boolean
 	// result is consumed directly as a predicate). The alternative-rounds driver
@@ -333,20 +324,6 @@ type PlanBuilder struct {
 	predicateMatchSeen bool
 }
 
-// HasNonViableFTSMatch reports whether the most recent build round saw a
-// predicate-context MATCH...AGAINST that could not be served by the native
-// FTSMysqlMatchAgainst builtin. The caller (optimize.go) uses this to
-// invalidate the round's plan and trigger the fts-like-fallback round.
-func (b *PlanBuilder) HasNonViableFTSMatch() bool {
-	return b.nonViableFTSMatch
-}
-
-// MarkNonViableFTSMatch records that a predicate-context MATCH...AGAINST in
-// the current build cannot be served natively. See HasNonViableFTSMatch.
-func (b *PlanBuilder) MarkNonViableFTSMatch() {
-	b.nonViableFTSMatch = true
-}
-
 // HasPredicateMatch reports whether the most recent build round saw a
 // direct-boolean-context MATCH...AGAINST. The caller (optimize.go) uses this
 // to decide whether to run the fts-like-fallback round for cost competition,

pkg/planner/optimize.go

@@ -581,7 +581,7 @@ func buildAndOptimizeLogicalPlanRound(
 		optFlag = optFlagAdjust(optFlag)
 	}
 	// when we are sure about we can have a ILIKE fallback for FTS function, we can
-	// directly ignore the error from AnalyzeTICIIndex phase.
+	// directly ignore the error from AnalyzeTICIIndex phase at first round.
 	finalPlan, cost, err := core.DoOptimize(ctx, sctx, optFlag, logic)
 	if err != nil {
 		if discard, fallbackErr := maybeArmFTSLikeFallback(sctx.GetSessionVars(), builder, err); discard {
@@ -598,17 +598,6 @@ func buildAndOptimizeLogicalPlanRound(
 	if builder.HasPredicateMatch() {
 		sctx.GetSessionVars().StmtCtx.AlternativeLogicalPlanHasPredicateContextMatch = true
 	}
-	// If this round saw a predicate-context MATCH that cannot be served by the
-	// native FTSMysqlMatchAgainst builtin, the produced plan would fail at
-	// execution. Discard it and arm AlternativeLogicalPlanFTSLikeFallback so
-	// any intervening rounds (correlate, etc.) re-rewrite with ILIKE too. The
-	// fts-like-fallback round below also forces this flag during setup; this
-	// outer assignment covers the non-viable case where the flag must stay
-	// true across all subsequent rounds, not just inside the LIKE round.
-	if builder.HasNonViableFTSMatch() {
-		sctx.GetSessionVars().StmtCtx.AlternativeLogicalPlanFTSLikeFallback = true
-		return p, names, false, nil
-	}
 
 	if *bestPlan == nil || cost < *bestCost {
 		*bestCost = cost
@@ -741,17 +730,14 @@ func optimize(ctx context.Context, sctx planctx.PlanContext, node *resolve.NodeW
 		initialLogicalPlanCtx = saveLogicalPlanBuildCtx(sessVars)
 		sessVars.StmtCtx.ResetAlternativeLogicalPlanSignals()
 		// Round 1 always uses the native FTSMysqlMatchAgainst builtin — same as
-		// the Alt-disabled default. The build records two signals on the
+		// the Alt-disabled default. The build records one signal on the
 		// planBuilder when MATCH...AGAINST is seen:
 		//   * HasPredicateMatch: any direct-boolean-context MATCH. The round
 		//     driver propagates this into stmtctx to trigger the
 		//     fts-like-fallback alternative round, which builds a competing
-		//     ILIKE-based plan; the cheaper of the two wins.
-		//   * HasNonViableFTSMatch: a predicate-context MATCH whose native form
-		//     cannot execute (no FTS index / no TiFlash replica / unsupported
-		//     modifier). The round driver discards round 1's plan and forces
-		//     AlternativeLogicalPlanFTSLikeFallback true so all subsequent
-		//     rounds (correlate, etc.) re-rewrite with ILIKE.
+		//     ILIKE-based plan; the cheaper of the two wins. If the real native
+		//     planning path later fails, maybeArmFTSLikeFallback forces the LIKE
+		//     round from the native error.
 	}
 
 	p, names, nonLogical, err := buildAndOptimizeLogicalPlanRound(

pkg/sessionctx/stmtctx/stmtctx.go

@@ -495,12 +495,10 @@ type StatementContext struct {
 	// Round 1 always runs with this flag false. The "fts-like-fallback"
 	// alternative round flips it to true (via its setup/cleanup) while it
 	// builds a competing ILIKE-based plan; the cost-cheapest plan wins via the
-	// normal alt-rounds cost comparison. If round 1's build records a
-	// predicate-context MATCH that cannot be served natively (no FTS index on a
-	// matched column / no TiFlash replica / modifier not pushdown-supported),
-	// optimize.go additionally invalidates round 1's plan and forces this flag
-	// true outside the round so any intervening rounds (correlate, etc.) also
-	// produce executable LIKE-based plans.
+	// normal alt-rounds cost comparison. If round 1's real native planning path
+	// fails with a fallbackable FTS error, optimize.go additionally forces this
+	// flag true outside the round so any intervening rounds (correlate, etc.)
+	// also produce executable LIKE-based plans.
 	AlternativeLogicalPlanFTSLikeFallback bool
 	// AlternativeLogicalPlanHasPredicateContextMatch indicates that round 1
 	// encountered a direct-boolean-context MATCH...AGAINST. The round driver