diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index 89acd8b87afd5c..a5c5f501a1ba0b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -19,7 +19,6 @@ import org.apache.doris.analysis.Analyzer; import org.apache.doris.analysis.SlotDescriptor; -import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TableSample; import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.analysis.TupleDescriptor; @@ -40,7 +39,6 @@ import org.apache.doris.datasource.hive.source.HiveScanNode; import org.apache.doris.datasource.hive.source.HiveSplit; import org.apache.doris.datasource.iceberg.source.IcebergSplit; -import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.qe.ConnectContext; import org.apache.doris.spi.Split; @@ -80,7 +78,6 @@ import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Set; /** * FileQueryScanNode for querying the file access type of catalog, now only support @@ -183,16 +180,6 @@ protected void initSchemaParams() throws UserException { params.setSrcTupleId(-1); } - /** - * Reset required_slots in contexts. This is called after Nereids planner do the projection. - * In the projection process, some slots may be removed. So call this to update the slots info. - */ - @Override - public void updateRequiredSlots(PlanTranslatorContext planTranslatorContext, - Set requiredByProjectSlotIdSet) throws UserException { - updateRequiredSlots(); - } - private void updateRequiredSlots() throws UserException { params.unsetRequiredSlots(); for (SlotDescriptor slot : desc.getSlots()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java index 8d5cf9cd764f71..70cbb405833021 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java @@ -27,7 +27,6 @@ import org.apache.doris.analysis.ExprSubstitutionMap; import org.apache.doris.analysis.FunctionCallExpr; import org.apache.doris.analysis.SlotDescriptor; -import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.Column; @@ -39,7 +38,6 @@ import org.apache.doris.common.UserException; import org.apache.doris.datasource.ExternalScanNode; import org.apache.doris.datasource.jdbc.JdbcExternalTable; -import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.qe.ConnectContext; import org.apache.doris.statistics.StatisticalType; @@ -59,7 +57,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; public class JdbcScanNode extends ExternalScanNode { @@ -259,12 +256,6 @@ public void finalizeForNereids() throws UserException { createScanRangeLocations(); } - @Override - public void updateRequiredSlots(PlanTranslatorContext context, Set requiredByProjectSlotIdSet) - throws UserException { - createJdbcColumns(); - } - @Override protected void createScanRangeLocations() throws UserException { scanRangeLocations = Lists.newArrayList(createSingleScanRangeLocations(backendPolicy)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/odbc/source/OdbcScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/odbc/source/OdbcScanNode.java index 059ac2cfc973f4..368a15bde48e3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/odbc/source/OdbcScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/odbc/source/OdbcScanNode.java @@ -22,7 +22,6 @@ import org.apache.doris.analysis.ExprSubstitutionMap; import org.apache.doris.analysis.FunctionCallExpr; import org.apache.doris.analysis.SlotDescriptor; -import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.Column; @@ -33,7 +32,6 @@ import org.apache.doris.common.UserException; import org.apache.doris.datasource.ExternalScanNode; import org.apache.doris.datasource.jdbc.source.JdbcScanNode; -import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.qe.ConnectContext; import org.apache.doris.statistics.StatisticalType; @@ -53,7 +51,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; /** @@ -118,12 +115,6 @@ public void finalizeForNereids() throws UserException { createScanRangeLocations(); } - @Override - public void updateRequiredSlots(PlanTranslatorContext context, Set requiredByProjectSlotIdSet) - throws UserException { - createOdbcColumns(); - } - @Override protected void createScanRangeLocations() throws UserException { scanRangeLocations = Lists.newArrayList(createSingleScanRangeLocations(backendPolicy)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index b5edcdee089285..aeecbd7eba2ede 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -17,7 +17,6 @@ package org.apache.doris.datasource.paimon.source; -import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.DdlException; @@ -27,7 +26,6 @@ import org.apache.doris.datasource.FileQueryScanNode; import org.apache.doris.datasource.paimon.PaimonExternalCatalog; import org.apache.doris.datasource.paimon.PaimonExternalTable; -import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.qe.ConnectContext; import org.apache.doris.spi.Split; @@ -38,7 +36,6 @@ import org.apache.doris.thrift.TFileType; import org.apache.doris.thrift.TPaimonDeletionFileDesc; import org.apache.doris.thrift.TPaimonFileDesc; -import org.apache.doris.thrift.TScanRangeLocations; import org.apache.doris.thrift.TTableFormatFileDesc; import com.google.common.base.Preconditions; @@ -279,22 +276,6 @@ private boolean supportNativeReader() { } } - //When calling 'setPaimonParams' and 'getSplits', the column trimming has not been performed yet, - // Therefore, paimon_column_names is temporarily reset here - @Override - public void updateRequiredSlots(PlanTranslatorContext planTranslatorContext, - Set requiredByProjectSlotIdSet) throws UserException { - super.updateRequiredSlots(planTranslatorContext, requiredByProjectSlotIdSet); - String cols = desc.getSlots().stream().map(slot -> slot.getColumn().getName()) - .collect(Collectors.joining(",")); - for (TScanRangeLocations tScanRangeLocations : scanRangeLocations) { - List ranges = tScanRangeLocations.scan_range.ext_scan_range.file_scan_range.ranges; - for (TFileRangeDesc tFileRangeDesc : ranges) { - tFileRangeDesc.table_format_params.paimon_params.setPaimonColumnNames(cols); - } - } - } - @Override public TFileType getLocationType() throws DdlException, MetaNotFoundException { return getLocationType(((FileStoreTable) source.getPaimonTable()).location().toString()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java index fd5a7db1f4ffc6..7afe8eed8c8b0e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java @@ -18,7 +18,6 @@ package org.apache.doris.datasource.trinoconnector.source; import org.apache.doris.analysis.SlotDescriptor; -import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; @@ -28,7 +27,6 @@ import org.apache.doris.datasource.FileQueryScanNode; import org.apache.doris.datasource.TableFormatType; import org.apache.doris.datasource.trinoconnector.TrinoConnectorPluginLoader; -import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.spi.Split; import org.apache.doris.statistics.StatisticalType; @@ -36,7 +34,6 @@ import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileRangeDesc; import org.apache.doris.thrift.TFileType; -import org.apache.doris.thrift.TScanRangeLocations; import org.apache.doris.thrift.TTableFormatFileDesc; import org.apache.doris.thrift.TTrinoConnectorFileDesc; import org.apache.doris.trinoconnector.TrinoColumnMetadata; @@ -321,31 +318,6 @@ private String encodeObjectToString(T t, ObjectMapperProvider objectMapperPr } } - // When calling 'setTrinoConnectorParams' and 'getSplits', the column trimming has not been performed yet, - // Therefore, trino_connector_column_names is temporarily reset here - @Override - public void updateRequiredSlots(PlanTranslatorContext planTranslatorContext, - Set requiredByProjectSlotIdSet) throws UserException { - super.updateRequiredSlots(planTranslatorContext, requiredByProjectSlotIdSet); - Map columnMetadataMap = source.getTargetTable().getColumnMetadataMap(); - Map columnHandleMap = source.getTargetTable().getColumnHandleMap(); - List columnHandles = new ArrayList<>(); - for (SlotDescriptor slotDescriptor : desc.getSlots()) { - String colName = slotDescriptor.getColumn().getName(); - if (columnMetadataMap.containsKey(colName)) { - columnHandles.add(columnHandleMap.get(colName)); - } - } - - for (TScanRangeLocations tScanRangeLocations : scanRangeLocations) { - List ranges = tScanRangeLocations.scan_range.ext_scan_range.file_scan_range.ranges; - for (TFileRangeDesc tFileRangeDesc : ranges) { - tFileRangeDesc.table_format_params.trino_connector_params.setTrinoConnectorColumnHandles( - encodeObjectToString(columnHandles, objectMapperProvider)); - } - } - } - @Override public TFileType getLocationType() throws DdlException, MetaNotFoundException { return getLocationType(""); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index cc4d8301238229..ff3de2249a6ccf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -46,8 +46,6 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; -import org.apache.doris.common.UserException; -import org.apache.doris.common.util.Util; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.es.source.EsScanNode; import org.apache.doris.datasource.hive.HMSExternalTable; @@ -276,6 +274,9 @@ public PlanFragment translatePlan(PhysicalPlan physicalPlan) { throw new AnalysisException("tables with unknown column stats: " + builder); } } + for (ScanNode scanNode : context.getScanNodes()) { + Utils.execWithUncheckedException(scanNode::finalizeForNereids); + } return rootFragment; } @@ -648,7 +649,6 @@ public PlanFragment visitPhysicalEsScan(PhysicalEsScan esScan, PlanTranslatorCon ) ); context.getTopnFilterContext().translateTarget(esScan, esScanNode, context); - Utils.execWithUncheckedException(esScanNode::finalizeForNereids); DataPartition dataPartition = DataPartition.RANDOM; PlanFragment planFragment = new PlanFragment(context.nextFragmentId(), esScanNode, dataPartition); context.addPlanFragment(planFragment); @@ -699,7 +699,6 @@ private PlanFragment getPlanFragmentForPhysicalFileScan(PhysicalFileScan fileSca ) ); context.getTopnFilterContext().translateTarget(fileScan, scanNode, context); - Utils.execWithUncheckedException(scanNode::finalizeForNereids); // Create PlanFragment DataPartition dataPartition = DataPartition.RANDOM; PlanFragment planFragment = createPlanFragment(scanNode, dataPartition, fileScan); @@ -725,7 +724,6 @@ public PlanFragment visitPhysicalJdbcScan(PhysicalJdbcScan jdbcScan, PlanTransla ) ); context.getTopnFilterContext().translateTarget(jdbcScan, jdbcScanNode, context); - Utils.execWithUncheckedException(jdbcScanNode::finalizeForNereids); DataPartition dataPartition = DataPartition.RANDOM; PlanFragment planFragment = new PlanFragment(context.nextFragmentId(), jdbcScanNode, dataPartition); context.addPlanFragment(planFragment); @@ -750,7 +748,6 @@ public PlanFragment visitPhysicalOdbcScan(PhysicalOdbcScan odbcScan, PlanTransla ) ); context.getTopnFilterContext().translateTarget(odbcScan, odbcScanNode, context); - Utils.execWithUncheckedException(odbcScanNode::finalizeForNereids); context.getTopnFilterContext().translateTarget(odbcScan, odbcScanNode, context); DataPartition dataPartition = DataPartition.RANDOM; PlanFragment planFragment = new PlanFragment(context.nextFragmentId(), odbcScanNode, dataPartition); @@ -829,8 +826,6 @@ public PlanFragment visitPhysicalOlapScan(PhysicalOlapScan olapScan, PlanTransla ); context.getTopnFilterContext().translateTarget(olapScan, olapScanNode, context); olapScanNode.setPushDownAggNoGrouping(context.getRelationPushAggOp(olapScan.getRelationId())); - // TODO: we need to remove all finalizeForNereids - olapScanNode.finalizeForNereids(); // Create PlanFragment // TODO: use a util function to convert distribution to DataPartition DataPartition dataPartition = DataPartition.RANDOM; @@ -917,7 +912,6 @@ public PlanFragment visitPhysicalSchemaScan(PhysicalSchemaScan schemaScan, PlanT .translateRuntimeFilterTarget(expr, finalScanNode, context) ) ); - Utils.execWithUncheckedException(scanNode::finalizeForNereids); context.addScanNode(scanNode, schemaScan); PlanFragment planFragment = createPlanFragment(scanNode, DataPartition.RANDOM, schemaScan); context.addPlanFragment(planFragment); @@ -939,7 +933,6 @@ public PlanFragment visitPhysicalTVFRelation(PhysicalTVFRelation tvfRelation, Pl .forEach(expr -> runtimeFilterGenerator.translateRuntimeFilterTarget(expr, scanNode, context) ) ); - Utils.execWithUncheckedException(scanNode::finalizeForNereids); context.addScanNode(scanNode, tvfRelation); // TODO: it is weird update label in this way @@ -2003,6 +1996,7 @@ public PlanFragment visitPhysicalProject(PhysicalProject project } requiredSlotIdSet.add(lastSlot.getId()); } + ((OlapScanNode) inputPlanNode).updateRequiredSlots(context, requiredByProjectSlotIdSet); } updateScanSlotsMaterialization((ScanNode) inputPlanNode, requiredSlotIdSet, requiredByProjectSlotIdSet, slotIdsByOrder, context); @@ -2452,22 +2446,16 @@ private void updateScanSlotsMaterialization(ScanNode scanNode, if (scanNode.getTupleDesc().getSlots().isEmpty()) { scanNode.getTupleDesc().getSlots().add(smallest); } - try { - if (context.getSessionVariable() != null - && context.getSessionVariable().forbidUnknownColStats - && !StatisticConstants.isSystemTable(scanNode.getTupleDesc().getTable())) { - for (SlotId slotId : requiredByProjectSlotIdSet) { - if (context.isColumnStatsUnknown(scanNode, slotId)) { - String colName = scanNode.getTupleDesc().getSlot(slotId.asInt()).getColumn().getName(); - throw new AnalysisException("meet unknown column stats: " + colName); - } + if (context.getSessionVariable() != null + && context.getSessionVariable().forbidUnknownColStats + && !StatisticConstants.isSystemTable(scanNode.getTupleDesc().getTable())) { + for (SlotId slotId : requiredByProjectSlotIdSet) { + if (context.isColumnStatsUnknown(scanNode, slotId)) { + String colName = scanNode.getTupleDesc().getSlot(slotId.asInt()).getColumn().getName(); + throw new AnalysisException("meet unknown column stats: " + colName); } - context.removeScanFromStatsUnknownColumnsMap(scanNode); } - scanNode.updateRequiredSlots(context, requiredByProjectSlotIdSet); - } catch (UserException e) { - Util.logAndThrowRuntimeException(LOG, - "User Exception while reset external file scan node contexts.", e); + context.removeScanFromStatsUnknownColumnsMap(scanNode); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index fb807c1ec78c45..ddd6c0f719e778 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -1749,7 +1749,11 @@ Set getDistributionColumnNames() { : Sets.newTreeSet(); } - @Override + /** + * Update required_slots in scan node contexts. This is called after Nereids planner do the projection. + * In the projection process, some slots may be removed. So call this to update the slots info. + * Currently, it is only used by ExternalFileScanNode, add the interface here to keep the Nereids code clean. + */ public void updateRequiredSlots(PlanTranslatorContext context, Set requiredByProjectSlotIdSet) { outputColumnUniqueIds.clear(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java index 224f85b37a567f..be02137cd1a9c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java @@ -48,7 +48,6 @@ import org.apache.doris.datasource.SplitAssignment; import org.apache.doris.datasource.SplitGenerator; import org.apache.doris.datasource.SplitSource; -import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.qe.ConnectContext; import org.apache.doris.rpc.RpcException; import org.apache.doris.statistics.StatisticalType; @@ -177,15 +176,6 @@ protected boolean isKeySearch() { return false; } - /** - * Update required_slots in scan node contexts. This is called after Nereids planner do the projection. - * In the projection process, some slots may be removed. So call this to update the slots info. - * Currently, it is only used by ExternalFileScanNode, add the interface here to keep the Nereids code clean. - */ - public void updateRequiredSlots(PlanTranslatorContext context, - Set requiredByProjectSlotIdSet) throws UserException { - } - private void computeColumnFilter(Column column, SlotDescriptor slotDesc, PartitionInfo partitionsInfo) { // Set `columnFilters` all the time because `DistributionPruner` also use this. // Maybe we could use `columnNameToRange` for `DistributionPruner` and