Skip to content

Commit 056ebac

Browse files
mohsakakasiafixin-zhang2
committed
Final analyzer/planner/optimizer changes for tvf except LocalExecutionPlanner and ExcludeColumns optimizer rule.
Co-authored-by: kasiafi <[email protected]> Co-authored-by: Xin Zhang <[email protected]>
1 parent f046e9f commit 056ebac

File tree

42 files changed

+5829
-128
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+5829
-128
lines changed

‎presto-analyzer/src/main/java/com/facebook/presto/sql/analyzer/Field.java‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -86,14 +86,6 @@ public Field(Optional<NodeLocation> nodeLocation, Optional<QualifiedName> relati
8686
this.aliased = aliased;
8787
}
8888

89-
public static Field newUnqualified(Optional<String> name, Type type)
90-
{
91-
requireNonNull(name, "name is null");
92-
requireNonNull(type, "type is null");
93-
94-
return new Field(Optional.empty(), Optional.empty(), name, type, false, Optional.empty(), Optional.empty(), false);
95-
}
96-
9789
public Optional<NodeLocation> getNodeLocation()
9890
{
9991
return nodeLocation;

‎presto-main-base/src/main/java/com/facebook/presto/sql/analyzer/StatementAnalyzer.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1457,7 +1457,7 @@ private void verifyRequiredColumns(TableFunctionInvocation node, Map<String, Lis
14571457
// the scope is recorded, because table arguments are already analyzed
14581458
Scope inputScope = analysis.getScope(tableArgumentsByName.get(name).getRelation());
14591459
columns.stream()
1460-
.filter(column -> column < 0 || column >= inputScope.getRelationType().getAllFieldCount()) // hidden columns can be required as well as visible columns
1460+
.filter(column -> column < 0 || column >= inputScope.getRelationType().getVisibleFieldCount())
14611461
.findFirst()
14621462
.ifPresent(column -> {
14631463
throw new SemanticException(TABLE_FUNCTION_IMPLEMENTATION_ERROR, "Invalid index: %s of required column from table argument %s", column, name);

‎presto-main-base/src/main/java/com/facebook/presto/sql/planner/BasePlanFragmenter.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@
4747
import com.facebook.presto.sql.planner.plan.SequenceNode;
4848
import com.facebook.presto.sql.planner.plan.SimplePlanRewriter;
4949
import com.facebook.presto.sql.planner.plan.StatisticsWriterNode;
50+
import com.facebook.presto.sql.planner.plan.TableFunctionNode;
51+
import com.facebook.presto.sql.planner.plan.TableFunctionProcessorNode;
5052
import com.facebook.presto.sql.planner.sanity.PlanChecker;
5153
import com.google.common.base.Preconditions;
5254
import com.google.common.collect.ImmutableList;
@@ -270,6 +272,22 @@ public PlanNode visitValues(ValuesNode node, RewriteContext<FragmentProperties>
270272
return context.defaultRewrite(node, context.get());
271273
}
272274

275+
@Override
276+
public PlanNode visitTableFunction(TableFunctionNode node, RewriteContext<FragmentProperties> context)
277+
{
278+
throw new IllegalStateException(format("Unexpected node: TableFunctionNode (%s)", node.getName()));
279+
}
280+
281+
@Override
282+
public PlanNode visitTableFunctionProcessor(TableFunctionProcessorNode node, RewriteContext<FragmentProperties> context)
283+
{
284+
if (!node.getSource().isPresent()) {
285+
// context is mutable. The leaf node should set the PartitioningHandle.
286+
context.get().addSourceDistribution(node.getId(), SOURCE_DISTRIBUTION, metadata, session);
287+
}
288+
return context.defaultRewrite(node, context.get());
289+
}
290+
273291
@Override
274292
public PlanNode visitExchange(ExchangeNode exchange, RewriteContext<FragmentProperties> context)
275293
{

‎presto-main-base/src/main/java/com/facebook/presto/sql/planner/PlanOptimizers.java‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
import com.facebook.presto.sql.planner.iterative.rule.ImplementBernoulliSampleAsFilter;
5151
import com.facebook.presto.sql.planner.iterative.rule.ImplementFilteredAggregations;
5252
import com.facebook.presto.sql.planner.iterative.rule.ImplementOffset;
53+
import com.facebook.presto.sql.planner.iterative.rule.ImplementTableFunctionSource;
5354
import com.facebook.presto.sql.planner.iterative.rule.InlineProjections;
5455
import com.facebook.presto.sql.planner.iterative.rule.InlineProjectionsOnValues;
5556
import com.facebook.presto.sql.planner.iterative.rule.InlineSqlFunctions;
@@ -81,6 +82,8 @@
8182
import com.facebook.presto.sql.planner.iterative.rule.PruneRedundantProjectionAssignments;
8283
import com.facebook.presto.sql.planner.iterative.rule.PruneSemiJoinColumns;
8384
import com.facebook.presto.sql.planner.iterative.rule.PruneSemiJoinFilteringSourceColumns;
85+
import com.facebook.presto.sql.planner.iterative.rule.PruneTableFunctionProcessorColumns;
86+
import com.facebook.presto.sql.planner.iterative.rule.PruneTableFunctionProcessorSourceColumns;
8487
import com.facebook.presto.sql.planner.iterative.rule.PruneTableScanColumns;
8588
import com.facebook.presto.sql.planner.iterative.rule.PruneTopNColumns;
8689
import com.facebook.presto.sql.planner.iterative.rule.PruneUpdateSourceColumns;
@@ -120,6 +123,7 @@
120123
import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantLimit;
121124
import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantSort;
122125
import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantSortColumns;
126+
import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantTableFunction;
123127
import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantTopN;
124128
import com.facebook.presto.sql.planner.iterative.rule.RemoveRedundantTopNColumns;
125129
import com.facebook.presto.sql.planner.iterative.rule.RemoveTrivialFilters;
@@ -310,6 +314,8 @@ public PlanOptimizers(
310314
new PruneValuesColumns(),
311315
new PruneWindowColumns(),
312316
new PruneLimitColumns(),
317+
new PruneTableFunctionProcessorColumns(),
318+
new PruneTableFunctionProcessorSourceColumns(),
313319
new PruneTableScanColumns());
314320

315321
builder.add(new LogicalCteOptimizer(metadata));
@@ -367,6 +373,14 @@ public PlanOptimizers(
367373
PlanOptimizer predicatePushDown = new StatsRecordingPlanOptimizer(optimizerStats, new PredicatePushDown(metadata, sqlParser, expressionOptimizerManager, featuresConfig.isNativeExecutionEnabled()));
368374
PlanOptimizer prefilterForLimitingAggregation = new StatsRecordingPlanOptimizer(optimizerStats, new PrefilterForLimitingAggregation(metadata, statsCalculator));
369375

376+
builder.add(
377+
new IterativeOptimizer(
378+
metadata,
379+
ruleStats,
380+
statsCalculator,
381+
costCalculator,
382+
ImmutableSet.of(new RewriteTableFunctionToTableScan(metadata))));
383+
370384
builder.add(
371385
new IterativeOptimizer(
372386
metadata,
@@ -409,6 +423,7 @@ public PlanOptimizers(
409423
.addAll(columnPruningRules)
410424
.addAll(ImmutableSet.of(
411425
new MergeDuplicateAggregation(metadata.getFunctionAndTypeManager()),
426+
new ImplementTableFunctionSource(metadata),
412427
new RemoveRedundantIdentityProjections(),
413428
new RemoveFullSample(),
414429
new EvaluateZeroSample(),
@@ -423,6 +438,7 @@ public PlanOptimizers(
423438
new PushLimitThroughSemiJoin(),
424439
new PushLimitThroughUnion(),
425440
new RemoveTrivialFilters(),
441+
new RemoveRedundantTableFunction(),
426442
new ImplementFilteredAggregations(metadata.getFunctionAndTypeManager()),
427443
new SingleDistinctAggregationToGroupBy(),
428444
new MultipleDistinctAggregationToMarkDistinct(),
@@ -762,7 +778,7 @@ public PlanOptimizers(
762778
ruleStats,
763779
statsCalculator,
764780
estimatedExchangesCostCalculator,
765-
ImmutableSet.of(new RemoveRedundantIdentityProjections(), new PruneRedundantProjectionAssignments())),
781+
ImmutableSet.of(new RemoveRedundantIdentityProjections(), new PruneRedundantProjectionAssignments(), new RemoveRedundantTableFunction())),
766782
new PushdownSubfields(metadata, expressionOptimizerManager));
767783

768784
builder.add(predicatePushDown); // Run predicate push down one more time in case we can leverage new information from layouts' effective predicate

‎presto-main-base/src/main/java/com/facebook/presto/sql/planner/PlannerUtils.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,9 @@ public static PlanNode addOverrideProjection(PlanNode source, PlanNodeIdAllocato
205205
|| source.getOutputVariables().stream().distinct().count() != source.getOutputVariables().size()) {
206206
return source;
207207
}
208+
if (source instanceof ProjectNode && ((ProjectNode) source).getAssignments().getMap().equals(variableMap)) {
209+
return source;
210+
}
208211
Assignments.Builder assignmentsBuilder = Assignments.builder();
209212
assignmentsBuilder.putAll(source.getOutputVariables().stream().collect(toImmutableMap(identity(), x -> variableMap.containsKey(x) ? variableMap.get(x) : x)));
210213
return new ProjectNode(source.getSourceLocation(), planNodeIdAllocator.getNextId(), source, assignmentsBuilder.build(), LOCAL);

‎presto-main-base/src/main/java/com/facebook/presto/sql/planner/QueryPlanner.java‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@
143143
import static java.lang.String.format;
144144
import static java.util.Objects.requireNonNull;
145145

146-
class QueryPlanner
146+
public class QueryPlanner
147147
{
148148
private final Analysis analysis;
149149
private final VariableAllocator variableAllocator;
@@ -524,7 +524,7 @@ private PlanBuilder project(PlanBuilder subPlan, Iterable<Expression> expression
524524
*
525525
* @return the new subplan and a mapping of each expression to the symbol representing the coercion or an existing symbol if a coercion wasn't needed
526526
*/
527-
private PlanAndMappings coerce(PlanBuilder subPlan, List<Expression> expressions, Analysis analysis, PlanNodeIdAllocator idAllocator, VariableAllocator variableAllocator, Metadata metadata)
527+
public PlanAndMappings coerce(PlanBuilder subPlan, List<Expression> expressions, Analysis analysis, PlanNodeIdAllocator idAllocator, VariableAllocator variableAllocator, Metadata metadata)
528528
{
529529
Assignments.Builder assignments = Assignments.builder();
530530
assignments.putAll(subPlan.getRoot().getOutputVariables().stream().collect(toImmutableMap(Function.identity(), Function.identity())));
@@ -1346,7 +1346,7 @@ private RowExpression rowExpression(Expression expression, SqlPlannerContext con
13461346
context.getTranslatorContext());
13471347
}
13481348

1349-
private static List<Expression> toSymbolReferences(List<VariableReferenceExpression> variables)
1349+
public static List<Expression> toSymbolReferences(List<VariableReferenceExpression> variables)
13501350
{
13511351
return variables.stream()
13521352
.map(variable -> new SymbolReference(
@@ -1355,6 +1355,11 @@ private static List<Expression> toSymbolReferences(List<VariableReferenceExpress
13551355
.collect(toImmutableList());
13561356
}
13571357

1358+
public static SymbolReference toSymbolReference(VariableReferenceExpression variable)
1359+
{
1360+
return new SymbolReference(variable.getSourceLocation().map(location -> new NodeLocation(location.getLine(), location.getColumn())), variable.getName());
1361+
}
1362+
13581363
public static class PlanAndMappings
13591364
{
13601365
private final PlanBuilder subPlan;

0 commit comments

Comments
 (0)