planner: add HasTiflash to avoid generating unnecessary mpp task | tidb-test=pr/2660

docs/note/planner/rule/rule_ai_notes.md

@@ -46,6 +46,35 @@ Test and verification:
 - Record with: `go test ./pkg/planner/core/casetest/rule -run TestConstantPropagateWithCollation -tags=intest,deadlock -record`.
 - Add integration test to `tests/integrationtest/t/select.test` and record via `pushd tests/integrationtest && ./run-tests.sh -r select && popd` (integration tests use `-r`, not `-record`).
 
+## 2026-02-12 - TiFlash `HasTiflash` propagation and plan-shape regression triage
+
+Background:
+- A refactor centralized `HasTiflash` propagation and changed `PreparePossibleProperties` behavior, which triggered broad plan output changes in TiFlash-related suites.
+- One real regression appeared in partition pruning: `Unknown column 'a' in expression` from `MakePartitionByFnCol` when the physical partition info was built from output names instead of real table column names.
+
+Key takeaways:
+- The primary fix should stay in the `PhysPlanPartInfo` producer path, not in ad-hoc downstream guards.
+- For partition pruning, always build pruning expressions with reconstructed table column names (`ReconstructTableColNames`) rather than post-projection/output names.
+- Avoid adding planner warning logs in this hot path for fallback behavior; keep the planning path deterministic and quiet unless there is actionable operator-level context.
+- When moving logic into `BaseLogicalPlan.PreparePossibleProperties`, verify both `p.hasTiflash` propagation and return-value semantics against operators that previously had custom implementations.
+
+Plan output triage checklist:
+1. Classify diffs before recording:
+- Plan ID only (`_5` vs `_6`) -> safe to update testdata.
+- Engine/mode change (`cop[tiflash]` vs `mpp[tiflash]`) -> validate session knobs and expected default behavior.
+- Structural/semantic shape change (new/removed operators or root/Mpp boundary shifts) -> treat as suspicious until proved intentional.
+2. For suspicious changes, compare the same case against `master` before re-recording.
+3. Keep lock-related expectations explicit:
+- `FOR UPDATE` can keep `SelectLock` at root while child subtree runs on MPP; this shape can be valid.
+4. Update only targeted result files after confirmation; avoid broad re-record without classification.
+
+Suggested validation set after this class of changes:
+- `go test -run TestTiflashPartitionTableScan --tags=intest ./pkg/executor/test/tiflashtest`
+- `go test -run TestTiFlashCostModel --tags=intest ./pkg/planner/core/casetest/cbotest`
+- `go test -run TestReadFromStorageHint --tags=intest ./pkg/planner/core/casetest/hint`
+- `go test -run TestPushDownToTiFlashWithKeepOrder --tags=intest ./pkg/planner/core/casetest/pushdown`
+- `go test -run TestPushDownProjectionForTiFlash --tags=intest ./pkg/planner/core/casetest/pushdown`
+
 ## 2026-02-16 - Index range dimension mismatch with appended handle column (issue #66291)
 
 Background:

pkg/executor/test/showtest/show_test.go

@@ -303,11 +303,12 @@ func TestShowWarningsForExprPushdown(t *testing.T) {
 	tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning|1105|Scalar function 'md5'(signature: MD5, return type: var_string(32)) is not supported to push down to tiflash now.", "Warning|1105|Aggregation can not be pushed to tiflash because arguments of AggFunc `max` contains unsupported exprs"))
 	tk.MustExec("explain format = 'brief' select /*+ read_from_storage(tiflash[show_warnings_expr_pushdown]) */ max(a) from show_warnings_expr_pushdown group by md5(value)")
 	require.Equal(t, uint16(2), tk.Session().GetSessionVars().StmtCtx.WarningCount())
-	tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning|1105|Scalar function 'md5'(signature: MD5, return type: var_string(32)) is not supported to push down to tiflash now.", "Warning|1105|Aggregation can not be pushed to tiflash because groupByItems contain unsupported exprs"))
+	tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|",
+		"Warning|1105|Scalar function 'md5'(signature: MD5, return type: var_string(32)) is not supported to push down to tiflash now.",
+		"Warning|1105|Aggregation can not be pushed to tiflash because groupByItems contain unsupported exprs"))
 	tk.MustExec("set tidb_opt_distinct_agg_push_down=0")
 	tk.MustExec("explain format = 'brief' select max(distinct a) from show_warnings_expr_pushdown group by value")
 	require.Equal(t, uint16(0), tk.Session().GetSessionVars().StmtCtx.WarningCount())
-	// tk.MustQuery("show warnings").Check(testkit.RowsWithSep("|", "Warning|1105|Aggregation can not be pushed to storage layer in non-mpp mode because it contains agg function with distinct"))
 }
 
 func TestShowGrantsPrivilege(t *testing.T) {

pkg/executor/testdata/slow_query_suite_out.json

@@ -20,35 +20,35 @@
       },
       {
         "SQL": "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where a > 1 order by f",
-        "Result": "[{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true},{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash)] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true},{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true}]"
+        "Result": "[{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash)] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true}]"
       },
       {
         "SQL": "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where f > 1",
-        "Result": "[{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true}]"
+        "Result": ""
       },
       {
         "SQL": "select /*+ READ_FROM_STORAGE(TIKV[t]) */ f from t where f > 1",
         "Result": "[{\"Level\":\"Note\",\"Message\":\"[t(tiflash),f,f_g] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true}]"
       },
       {
         "SQL": "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where f > 3 and g = 5",
-        "Result": "[{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true},{\"Level\":\"Note\",\"Message\":\"[t(tiflash),f_g,g] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true}]"
+        "Result": "[{\"Level\":\"Note\",\"Message\":\"[t(tiflash),f_g,g] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true}]"
       },
       {
         "SQL": "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where g = 5 order by f",
-        "Result": "[{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true},{\"Level\":\"Note\",\"Message\":\"[t(tiflash),g] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true},{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true},{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash),f_g,g] remain after pruning paths for t given Prop{SortItems: [{test.t.f asc}], TaskTp: rootTask}\",\"IsExtra\":true}]"
+        "Result": "[{\"Level\":\"Note\",\"Message\":\"[t(tiflash),g] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true},{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash),f_g,g] remain after pruning paths for t given Prop{SortItems: [{test.t.f asc}], TaskTp: rootTask}\",\"IsExtra\":true}]"
       },
       {
         "SQL": "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where d = 3 order by c, e",
-        "Result": "[{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true},{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash)] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true},{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true}]"
+        "Result": "[{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash)] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true}]"
       },
       {
         "SQL": "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where h = '11,22'",
-        "Result": "[{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true},{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true},{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash)] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true}]"
+        "Result": "[{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash)] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true}]"
       },
       {
         "SQL": "select /*+ READ_FROM_STORAGE(TIKV[t]) */ * from t where a > rand()*100",
-        "Result": "[{\"Level\":\"Warning\",\"Message\":\"Scalar function 'rand'(signature: Rand, return type: double) is not supported to push down to storage layer now.\",\"IsExtra\":true},{\"Level\":\"Warning\",\"Message\":\"Expression about 'test.t.h' can not be pushed to TiFlash because it contains unsupported calculation of type 'set'.\",\"IsExtra\":true},{\"Level\":\"Warning\",\"Message\":\"Scalar function 'rand'(signature: Rand, return type: double) is not supported to push down to tiflash now.\",\"IsExtra\":true},{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash)] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true}]"
+        "Result": "[{\"Level\":\"Warning\",\"Message\":\"Scalar function 'rand'(signature: Rand, return type: double) is not supported to push down to storage layer now.\",\"IsExtra\":true},{\"Level\":\"Note\",\"Message\":\"[t,t(tiflash)] remain after pruning paths for t given Prop{SortItems: [], TaskTp: rootTask}\",\"IsExtra\":true}]"
       }
     ]
   }

pkg/planner/cascades/memo/group_expr.go

@@ -223,19 +223,23 @@ func (e *GroupExpression) DeriveLogicalProp() (err error) {
 	}
 	childStats := make([]*property.StatsInfo, 0, len(e.Inputs))
 	childSchema := make([]*expression.Schema, 0, len(e.Inputs))
-	childProperties := make([][][]*expression.Column, 0, len(e.Inputs))
+	childProperties := make([]*base.PossiblePropertiesInfo, 0, len(e.Inputs))
 	for _, childG := range e.Inputs {
 		childGProp := childG.GetLogicalProperty()
 		childStats = append(childStats, childGProp.Stats)
 		childSchema = append(childSchema, childGProp.Schema)
-		childProperties = append(childProperties, childGProp.PossibleProps)
+		childProperties = append(childProperties, &base.PossiblePropertiesInfo{
+			Orders:     childGProp.PossibleProps,
+			HasTiflash: childGProp.HasTiflash,
+		})
 	}
 	e.GetGroup().SetLogicalProperty(property.NewLogicalProp())
 	// currently the schemaProducer side logical op is still useful for group schema.
 	tmpFD := e.LogicalPlan.GetBaseLogicalPlan().(*logicalop.BaseLogicalPlan).FDs()
 	tmpSchema := e.LogicalPlan.Schema()
 	tmpStats := e.LogicalPlan.StatsInfo()
 	var tmpPossibleProps [][]*expression.Column
+	var tmpHasTiflash bool
 	// the leaves node may have already had their stats in join reorder est phase, while
 	// their group ndv signal is passed in CollectPredicateColumnsPoint which is applied
 	// behind join reorder rule, we should build their group ndv again (implied in DeriveStats).
@@ -252,12 +256,17 @@ func (e *GroupExpression) DeriveLogicalProp() (err error) {
 		// todo: extractFD should be refactored as take in childFDs, and return the new FDSet rather than depend on tree.
 		tmpFD = e.LogicalPlan.ExtractFD()
 		// prepare the possible sort columns for the group, which require fillIndexPath to fill index cols.
-		tmpPossibleProps = e.LogicalPlan.PreparePossibleProperties(tmpSchema, childProperties...)
+		tmp := e.LogicalPlan.PreparePossibleProperties(tmpSchema, childProperties...)
+		if tmp != nil {
+			tmpPossibleProps = tmp.Orders
+			tmpHasTiflash = tmp.HasTiflash
+		}
 	}
 	e.GetGroup().GetLogicalProperty().Schema = tmpSchema
 	e.GetGroup().GetLogicalProperty().Stats = tmpStats
 	e.GetGroup().GetLogicalProperty().FD = tmpFD
 	e.GetGroup().GetLogicalProperty().PossibleProps = tmpPossibleProps
+	e.GetGroup().GetLogicalProperty().HasTiflash = tmpHasTiflash
 	return nil
 }
 

pkg/planner/cascades/old/optimize.go

@@ -246,7 +246,7 @@ func (opt *Optimizer) onPhaseImplementation(_ base.PlanContext, g *memo.Group) (
 	prop := &property.PhysicalProperty{
 		ExpectedCnt: math.MaxFloat64,
 	}
-	preparePossibleProperties(g, make(map[*memo.Group][][]*expression.Column))
+	preparePossibleProperties(g, make(map[*memo.Group]*base.PossiblePropertiesInfo))
 	// TODO replace MaxFloat64 costLimit by variable from sctx, or other sources.
 	impl, err := opt.implGroup(g, prop, math.MaxFloat64)
 	if err != nil {
@@ -355,18 +355,28 @@ func (opt *Optimizer) implGroupExpr(cur *memo.GroupExpr, reqPhysProp *property.P
 // preparePossibleProperties recursively calls LogicalPlan PreparePossibleProperties
 // interface. It will fulfill the the possible properties fields of LogicalAggregation
 // and LogicalJoin.
-func preparePossibleProperties(g *memo.Group, propertyMap map[*memo.Group][][]*expression.Column) [][]*expression.Column {
+func preparePossibleProperties(g *memo.Group, propertyMap map[*memo.Group]*base.PossiblePropertiesInfo) *base.PossiblePropertiesInfo {
 	if prop, ok := propertyMap[g]; ok {
 		return prop
 	}
 	groupPropertyMap := make(map[string][]*expression.Column)
+	groupHasTiflash := false
 	for elem := g.Equivalents.Front(); elem != nil; elem = elem.Next() {
 		expr := elem.Value.(*memo.GroupExpr)
-		childrenProperties := make([][][]*expression.Column, len(expr.Children))
+		childrenProperties := make([]*base.PossiblePropertiesInfo, len(expr.Children))
 		for i, child := range expr.Children {
-			childrenProperties[i] = preparePossibleProperties(child, propertyMap)
+			childProps := preparePossibleProperties(child, propertyMap)
+			if childProps == nil {
+				childProps = &base.PossiblePropertiesInfo{}
+			}
+			childrenProperties[i] = childProps
+		}
+		exprInfo := expr.ExprNode.PreparePossibleProperties(expr.Schema(), childrenProperties...)
+		if exprInfo == nil {
+			continue
 		}
-		exprProperties := expr.ExprNode.PreparePossibleProperties(expr.Schema(), childrenProperties...)
+		groupHasTiflash = groupHasTiflash || exprInfo.HasTiflash
+		exprProperties := exprInfo.Orders
 		for _, newPropCols := range exprProperties {
 			// Check if the prop has already been in `groupPropertyMap`.
 			newProp := property.PhysicalProperty{SortItems: property.SortItemsFromCols(newPropCols, true)}
@@ -380,6 +390,10 @@ func preparePossibleProperties(g *memo.Group, propertyMap map[*memo.Group][][]*e
 	for _, prop := range groupPropertyMap {
 		resultProps = append(resultProps, prop)
 	}
-	propertyMap[g] = resultProps
-	return resultProps
+	result := &base.PossiblePropertiesInfo{
+		Orders:     resultProps,
+		HasTiflash: groupHasTiflash,
+	}
+	propertyMap[g] = result
+	return result
 }

pkg/planner/cascades/old/optimize_test.go

@@ -179,18 +179,18 @@ func TestPreparePossibleProperties(t *testing.T) {
 	//   IndexScan_8 table:t, index:f, g
 	// Group#4 Schema:[test.t.a,test.t.f]
 	//   IndexScan_6 table:t, index:f
-	propMap := make(map[*memo.Group][][]*expression.Column)
+	propMap := make(map[*memo.Group]*base.PossiblePropertiesInfo)
 	aggProp := preparePossibleProperties(group, propMap)
 	// We only have one prop for Group0 : f
-	require.Len(t, aggProp, 1)
-	require.True(t, aggProp[0][0].EqualColumn(columnF))
+	require.Len(t, aggProp.Orders, 1)
+	require.True(t, aggProp.Orders[0][0].EqualColumn(columnF))
 
 	gatherGroup := group.Equivalents.Front().Value.(*memo.GroupExpr).Children[0]
 	gatherProp, ok := propMap[gatherGroup]
 	require.True(t, ok)
 	// We have 2 props for Group1: [f], [a]
-	require.Len(t, gatherProp, 2)
-	for _, prop := range gatherProp {
+	require.Len(t, gatherProp.Orders, 2)
+	for _, prop := range gatherProp.Orders {
 		require.Len(t, prop, 1)
 		require.True(t, prop[0].EqualColumn(columnA) || prop[0].EqualColumn(columnF))
 	}

pkg/planner/core/base/plan_base.go

@@ -224,7 +224,7 @@ type LogicalPlan interface {
 
 	// PreparePossibleProperties is only used for join and aggregation. Like group by a,b,c, all permutation of (a,b,c) is
 	// valid, but the ordered indices in leaf plan is limited. So we can get all possible order properties by a pre-walking.
-	PreparePossibleProperties(schema *expression.Schema, childrenProperties ...[][]*expression.Column) [][]*expression.Column
+	PreparePossibleProperties(schema *expression.Schema, childrenProperties ...*PossiblePropertiesInfo) *PossiblePropertiesInfo
 
 	// ExtractCorrelatedCols extracts correlated columns inside the LogicalPlan.
 	ExtractCorrelatedCols() []*expression.CorrelatedColumn
@@ -378,3 +378,63 @@ type PhysicalJoin interface {
 	GetInnerChildIdx() int
 	GetJoinType() JoinType
 }
+
+// PossiblePropertiesInfo is used to store all possible order properties.
+type PossiblePropertiesInfo struct {
+	// all possible order properties
+	Orders [][]*expression.Column
+	// HasTiflash is a runtime pruning signal and is intentionally excluded from hash/equals.
+	HasTiflash bool
+}
+
+// Hash64 implements the HashEquals interface.
+func (info *PossiblePropertiesInfo) Hash64(h base.Hasher) {
+	if info == nil {
+		h.HashByte(base.NilFlag)
+		return
+	}
+	h.HashByte(base.NotNilFlag)
+	if info.Orders == nil {
+		h.HashByte(base.NilFlag)
+	} else {
+		h.HashByte(base.NotNilFlag)
+		h.HashInt(len(info.Orders))
+		for _, one := range info.Orders {
+			h.HashInt(len(one))
+			for _, col := range one {
+				col.Hash64(h)
+			}
+		}
+	}
+}
+
+// Equals implements the HashEquals interface.
+func (info *PossiblePropertiesInfo) Equals(other any) bool {
+	info2, ok := other.(*PossiblePropertiesInfo)
+	if !ok {
+		return false
+	}
+	if info == nil {
+		return info2 == nil
+	}
+	if info2 == nil {
+		return false
+	}
+	if (info.Orders == nil && info2.Orders != nil) || (info.Orders != nil && info2.Orders == nil) || len(info.Orders) != len(info2.Orders) {
+		return false
+	}
+	for i, one := range info.Orders {
+		if len(one) != len(info2.Orders[i]) {
+			return false
+		}
+		for j, col := range one {
+			if (col == nil && info2.Orders[i][j] != nil) || (col != nil && info2.Orders[i][j] == nil) {
+				return false
+			}
+			if col != nil && !col.Equals(info2.Orders[i][j]) {
+				return false
+			}
+		}
+	}
+	return true
+}