diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneTableScanColumns.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneTableScanColumns.java index f0969147fa58..bf02d62f546e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneTableScanColumns.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/iterative/rule/PruneTableScanColumns.java @@ -28,7 +28,9 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.AggregationTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.InformationSchemaTableScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeAlignedDeviceViewScanNode; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeDeviceViewScanNode; +import org.apache.iotdb.db.queryengine.plan.relational.planner.node.TreeNonAlignedDeviceViewScanNode; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -91,7 +93,51 @@ public static Optional pruneColumns(TableScanNode node, Set re .forEach( symbol -> newAssignments.put(symbol, node.getAssignments().get(symbol)))); - if (node instanceof TreeDeviceViewScanNode) { + if (node instanceof TreeAlignedDeviceViewScanNode) { + TreeAlignedDeviceViewScanNode treeDeviceViewScanNode = + (TreeAlignedDeviceViewScanNode) deviceTableScanNode; + TreeAlignedDeviceViewScanNode prunedNode = + new TreeAlignedDeviceViewScanNode( + deviceTableScanNode.getPlanNodeId(), + deviceTableScanNode.getQualifiedObjectName(), + newOutputs, + newAssignments, + deviceTableScanNode.getDeviceEntries(), + deviceTableScanNode.getTagAndAttributeIndexMap(), + deviceTableScanNode.getScanOrder(), + deviceTableScanNode.getTimePredicate().orElse(null), + deviceTableScanNode.getPushDownPredicate(), + deviceTableScanNode.getPushDownLimit(), + deviceTableScanNode.getPushDownOffset(), + deviceTableScanNode.isPushLimitToEachDevice(), + deviceTableScanNode.containsNonAlignedDevice(), + treeDeviceViewScanNode.getTreeDBName(), + treeDeviceViewScanNode.getMeasurementColumnNameMap()); + prunedNode.setRegionReplicaSet(deviceTableScanNode.getRegionReplicaSet()); + return Optional.of(prunedNode); + } else if (node instanceof TreeNonAlignedDeviceViewScanNode) { + TreeNonAlignedDeviceViewScanNode treeDeviceViewScanNode = + (TreeNonAlignedDeviceViewScanNode) deviceTableScanNode; + TreeNonAlignedDeviceViewScanNode prunedNode = + new TreeNonAlignedDeviceViewScanNode( + deviceTableScanNode.getPlanNodeId(), + deviceTableScanNode.getQualifiedObjectName(), + newOutputs, + newAssignments, + deviceTableScanNode.getDeviceEntries(), + deviceTableScanNode.getTagAndAttributeIndexMap(), + deviceTableScanNode.getScanOrder(), + deviceTableScanNode.getTimePredicate().orElse(null), + deviceTableScanNode.getPushDownPredicate(), + deviceTableScanNode.getPushDownLimit(), + deviceTableScanNode.getPushDownOffset(), + deviceTableScanNode.isPushLimitToEachDevice(), + deviceTableScanNode.containsNonAlignedDevice(), + treeDeviceViewScanNode.getTreeDBName(), + treeDeviceViewScanNode.getMeasurementColumnNameMap()); + prunedNode.setRegionReplicaSet(deviceTableScanNode.getRegionReplicaSet()); + return Optional.of(prunedNode); + } else if (node instanceof TreeDeviceViewScanNode) { TreeDeviceViewScanNode treeDeviceViewScanNode = (TreeDeviceViewScanNode) deviceTableScanNode; return Optional.of( @@ -112,7 +158,7 @@ public static Optional pruneColumns(TableScanNode node, Set re treeDeviceViewScanNode.getTreeDBName(), treeDeviceViewScanNode.getMeasurementColumnNameMap())); } else { - return Optional.of( + DeviceTableScanNode prunedNode = new DeviceTableScanNode( deviceTableScanNode.getPlanNodeId(), deviceTableScanNode.getQualifiedObjectName(), @@ -126,7 +172,9 @@ public static Optional pruneColumns(TableScanNode node, Set re deviceTableScanNode.getPushDownLimit(), deviceTableScanNode.getPushDownOffset(), deviceTableScanNode.isPushLimitToEachDevice(), - deviceTableScanNode.containsNonAlignedDevice())); + deviceTableScanNode.containsNonAlignedDevice()); + prunedNode.setRegionReplicaSet(deviceTableScanNode.getRegionReplicaSet()); + return Optional.of(prunedNode); } } else if (node instanceof InformationSchemaTableScanNode) { // For the convenience of process in execution stage, column-prune for diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java index 8df10d5d8ec1..b5ed662cdae4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/optimizations/SortElimination.java @@ -25,14 +25,21 @@ import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.FillNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.GapFillNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.PatternRecognitionNode; +import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.ProjectNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.SortNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.StreamSortNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.ValueFillNode; import org.apache.iotdb.commons.queryengine.plan.relational.planner.node.WindowNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanVisitor; +import org.apache.iotdb.db.queryengine.plan.relational.planner.iterative.rule.PruneTableScanColumns; import org.apache.iotdb.db.queryengine.plan.relational.planner.node.DeviceTableScanNode; +import com.google.common.collect.ImmutableSet; + import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Set; /** * Optimization phase: Distributed plan planning. @@ -42,6 +49,8 @@ * SortNode can be eliminated. *
  • When order by all IDColumns and time, the SortNode can be eliminated. *
  • When StreamSortIndex==OrderBy size()-1, remove this StreamSortNode + *
  • After SortNode elimination, visitProject will remove redundant identity ProjectNodes above + * TableScan and pushes column pruning into the scan. */ public class SortElimination implements PlanOptimizer { @@ -64,6 +73,43 @@ public PlanNode visitPlan(PlanNode node, Context context) { return newNode; } + @Override + public PlanNode visitProject(ProjectNode node, Context context) { + Context newContext = new Context(); + PlanNode child = node.getChild().accept(this, newContext); + context.setCannotEliminateSort(newContext.cannotEliminateSort); + + // Remove useless ProjectNode and prune columns of TableScanNode + return eliminateProjectOverTableScan(node, child) + .orElseGet(() -> node.replaceChildren(Collections.singletonList(child))); + } + + private static Optional eliminateProjectOverTableScan( + ProjectNode project, PlanNode child) { + if (!(child instanceof DeviceTableScanNode) || !project.isIdentity()) { + return Optional.empty(); + } + + // Notice that SortNode may have been eliminated in TableDistributedPlanGenerator + DeviceTableScanNode tableScan = (DeviceTableScanNode) child; + int projectOutputsSize = project.getOutputSymbols().size(); + int scanOutputsSize = tableScan.getOutputSymbols().size(); + if (projectOutputsSize > scanOutputsSize) { + return Optional.empty(); + } + + List projectOutputs = project.getOutputSymbols(); + Set scanOutputs = ImmutableSet.copyOf(tableScan.getOutputSymbols()); + if (!scanOutputs.containsAll(projectOutputs)) { + return Optional.empty(); + } + + if (projectOutputsSize == scanOutputsSize) { + return Optional.of(tableScan); + } + return PruneTableScanColumns.pruneColumns(tableScan, ImmutableSet.copyOf(projectOutputs)); + } + @Override public PlanNode visitSort(SortNode node, Context context) { Context newContext = new Context(); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/SortTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/SortTest.java index 40797da29a90..f7b393b724a2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/SortTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/SortTest.java @@ -35,6 +35,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.LogicalQueryPlan; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.sink.IdentitySinkNode; import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; +import org.apache.iotdb.db.queryengine.plan.relational.planner.PlanTester; import org.apache.iotdb.db.queryengine.plan.relational.planner.SymbolAllocator; import org.apache.iotdb.db.queryengine.plan.relational.planner.TableLogicalPlanner; import org.apache.iotdb.db.queryengine.plan.relational.planner.distribute.TableDistributedPlanner; @@ -42,6 +43,8 @@ import org.apache.iotdb.db.queryengine.plan.relational.planner.node.ExchangeNode; import org.apache.iotdb.db.queryengine.plan.statement.component.Ordering; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import org.junit.BeforeClass; import org.junit.Test; @@ -58,6 +61,12 @@ import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.TestUtils.TEST_MATADATA; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.TestUtils.assertTableScan; import static org.apache.iotdb.db.queryengine.plan.relational.analyzer.TestUtils.getChildrenNode; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanAssert.assertPlan; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.exchange; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.mergeSort; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.output; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.project; +import static org.apache.iotdb.db.queryengine.plan.relational.planner.assertions.PlanMatchPattern.tableScan; import static org.apache.iotdb.db.queryengine.plan.statement.component.Ordering.ASC; import static org.apache.iotdb.db.queryengine.plan.statement.component.Ordering.DESC; import static org.junit.Assert.assertEquals; @@ -767,4 +776,39 @@ public void assertTopKNoFilter( expectedPushDownOffset, isPushLimitToEachDevice); } + + @Test + public void singleDeviceOrderByAllIdsAndTimeDescTest() { + PlanTester planTester = new PlanTester(); + planTester.createPlan( + "SELECT s1, s2, s3 FROM table1 " + + "WHERE time >= 1000 AND time <= 2000 AND tag1='beijing' AND tag2='A1'" + + "ORDER BY tag1, tag2, tag3, time DESC"); + assertPlan( + planTester.getFragmentPlan(0), + output( + tableScan( + "testdb.table1", + ImmutableList.of("s1", "s2", "s3"), + ImmutableSet.of("time", "s1", "s2", "s3")))); + } + + @Test + public void multiDeviceOrderByAllIdsAndTimeDescTest() { + PlanTester planTester = new PlanTester(); + planTester.createPlan( + "SELECT s1, s2, s3 FROM table1 " + + "WHERE time >= 1000 AND time <= 2000 AND tag1='beijing' " + + "ORDER BY tag1, tag2, tag3, time DESC"); + assertPlan( + planTester.getFragmentPlan(0), + output(project(mergeSort(exchange(), exchange(), exchange())))); + // Device in multi-region + assertPlan( + planTester.getFragmentPlan(1), + tableScan( + "testdb.table1", + ImmutableList.of("time", "tag1", "tag2", "tag3", "s1", "s2", "s3"), + ImmutableSet.of("time", "tag1", "tag2", "tag3", "s1", "s2", "s3"))); + } }