diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/BufferedVectorizedRowBatch.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/BufferedVectorizedRowBatch.java index f70fc2dea470..2e750f6408dc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/BufferedVectorizedRowBatch.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/BufferedVectorizedRowBatch.java @@ -24,7 +24,6 @@ */ public class BufferedVectorizedRowBatch extends VectorizedRowBatch { boolean isLastGroupBatch; - boolean isInputExpressionEvaluated; public BufferedVectorizedRowBatch(int numCols) { super(numCols); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorBase.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorBase.java index 9243c3cb00e0..31995c434d8d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorBase.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorBase.java @@ -86,12 +86,6 @@ public void setNullsLast(boolean nullsLast) { this.nullsLast = nullsLast; } - // Evaluate the aggregation input argument expression. - public void evaluateInputExpr(VectorizedRowBatch batch) throws HiveException { - if (inputVecExpr != null) { - inputVecExpr.evaluate(batch); - } - } /* * Evaluate the aggregation over one of the group's batches. In this mode, evaluator is not diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCount.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCount.java index 94b038e2e8ee..eed564b39717 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCount.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCount.java @@ -46,7 +46,6 @@ public VectorPTFEvaluatorCount(WindowFrameDef windowFrameDef, VectorExpression i public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Count non-null column rows. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountDistinct.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountDistinct.java index 167c07dc209c..3f34274cf015 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountDistinct.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountDistinct.java @@ -68,7 +68,6 @@ public VectorPTFEvaluatorCountDistinct(WindowFrameDef windowFrameDef, @Override public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // We do not filter when PTF is in reducer. Preconditions.checkState(!batch.selectedInUse); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountStar.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountStar.java index c48141ef8d12..3cff308550c4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountStar.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorCountStar.java @@ -42,7 +42,6 @@ public VectorPTFEvaluatorCountStar(WindowFrameDef windowFrameDef, VectorExpressi @Override public void evaluateGroupBatch(VectorizedRowBatch batch) { // No input expression for COUNT(*). - // evaluateInputExpr(batch); // Count all rows. count += batch.size; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalAvg.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalAvg.java index 5d4d757060d3..7f7cfec9d0dd 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalAvg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalAvg.java @@ -54,7 +54,6 @@ public VectorPTFEvaluatorDecimalAvg(WindowFrameDef windowFrameDef, VectorExpress public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Sum all non-null decimal column values for avg; maintain isGroupResultNull; after last row of // last group batch compute the group avg when sum is non-null. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalFirstValue.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalFirstValue.java index d21e882084b7..c500cf193ec9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalFirstValue.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalFirstValue.java @@ -50,7 +50,6 @@ public VectorPTFEvaluatorDecimalFirstValue(WindowFrameDef windowFrameDef, public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // First row determines isGroupResultNull and decimal firstValue; stream fill result as repeated. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalLastValue.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalLastValue.java index 0fb88d69c3da..78ff8b7bd200 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalLastValue.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalLastValue.java @@ -50,7 +50,6 @@ public VectorPTFEvaluatorDecimalLastValue(WindowFrameDef windowFrameDef, public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Last row of last batch determines isGroupResultNull and decimal lastValue. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMax.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMax.java index 11f146db706b..33a8a8b4ddb8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMax.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMax.java @@ -48,7 +48,6 @@ public VectorPTFEvaluatorDecimalMax(WindowFrameDef windowFrameDef, VectorExpress public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Determine maximum of all non-null decimal column values; maintain isGroupResultNull. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMin.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMin.java index e06d18688565..bb57159338fc 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalMin.java @@ -46,7 +46,6 @@ public VectorPTFEvaluatorDecimalMin(WindowFrameDef windowFrameDef, VectorExpress public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Determine minimum of all non-null decimal column values; maintain isGroupResultNull. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalSum.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalSum.java index f9263e17af93..02b388d55dcb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalSum.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDecimalSum.java @@ -48,7 +48,6 @@ public VectorPTFEvaluatorDecimalSum(WindowFrameDef windowFrameDef, VectorExpress public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Sum all non-null decimal column values; maintain isGroupResultNull. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleAvg.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleAvg.java index d7123fb25799..7cd42a382c23 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleAvg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleAvg.java @@ -46,7 +46,6 @@ public VectorPTFEvaluatorDoubleAvg(WindowFrameDef windowFrameDef, VectorExpressi public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Sum all non-null double column values for avg; maintain isGroupResultNull; after last row of // last group batch compute the group avg when sum is non-null. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleFirstValue.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleFirstValue.java index 40b49d7960c6..026baa0aa330 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleFirstValue.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleFirstValue.java @@ -49,7 +49,6 @@ public VectorPTFEvaluatorDoubleFirstValue(WindowFrameDef windowFrameDef, public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // First row determines isGroupResultNull and double firstValue; stream fill result as repeated. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleLastValue.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleLastValue.java index e7e4a0ce3a7f..d89364136788 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleLastValue.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleLastValue.java @@ -47,7 +47,6 @@ public VectorPTFEvaluatorDoubleLastValue(WindowFrameDef windowFrameDef, public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Last row of last batch determines isGroupResultNull and double lastValue. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMax.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMax.java index 5b124d20269a..1a8f029639d1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMax.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMax.java @@ -45,7 +45,6 @@ public VectorPTFEvaluatorDoubleMax(WindowFrameDef windowFrameDef, VectorExpressi public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Determine maximum of all non-null double column values; maintain isGroupResultNull. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMin.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMin.java index b76df013b351..e8f28eb9c312 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleMin.java @@ -45,7 +45,6 @@ public VectorPTFEvaluatorDoubleMin(WindowFrameDef windowFrameDef, VectorExpressi public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Determine minimum of all non-null double column values; maintain isGroupResultNull. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleSum.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleSum.java index e8cda454bcd0..4ecefb4f2fe3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleSum.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorDoubleSum.java @@ -42,7 +42,6 @@ public VectorPTFEvaluatorDoubleSum(WindowFrameDef windowFrameDef, VectorExpressi public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Sum all non-null double column values; maintain isGroupResultNull. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLag.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLag.java index d35d089c6c0e..319225088627 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLag.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLag.java @@ -38,7 +38,7 @@ public Object runOnRange(int rowNum, Range range, VectorPTFGroupBatches batches) if (lagRow < 0 || lagRow >= batches.size()) { return getDefaultValue(rowNum, batches); } else { - return batches.getValueAndEvaluateInputExpression(this, lagRow, inputColumnNum); + return batches.getValue(lagRow, inputColumnNum); } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLead.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLead.java index db0332ce9d93..d2806bb65b91 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLead.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLead.java @@ -38,7 +38,7 @@ public Object runOnRange(int rowNum, Range range, VectorPTFGroupBatches batches) if (leadRow < 0 || leadRow >= batches.size()) { return getDefaultValue(rowNum, batches); } else { - return batches.getValueAndEvaluateInputExpression(this, leadRow, inputColumnNum); + return batches.getValue(leadRow, inputColumnNum); } } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongAvg.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongAvg.java index 9c5b402fa5c6..5729a9a11e22 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongAvg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongAvg.java @@ -46,7 +46,6 @@ public VectorPTFEvaluatorLongAvg(WindowFrameDef windowFrameDef, VectorExpression public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Sum all non-null long column values for avg; maintain isGroupResultNull; after last row of // last group batch compute the group avg when sum is non-null. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongFirstValue.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongFirstValue.java index 1b1ca2049395..d370a07880d7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongFirstValue.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongFirstValue.java @@ -49,7 +49,6 @@ public VectorPTFEvaluatorLongFirstValue(WindowFrameDef windowFrameDef, public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // First row determines isGroupResultNull and long firstValue; stream fill result as repeated. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongLastValue.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongLastValue.java index bf0f27af832b..536c05c99012 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongLastValue.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongLastValue.java @@ -48,7 +48,6 @@ public VectorPTFEvaluatorLongLastValue(WindowFrameDef windowFrameDef, public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Last row of last batch determines isGroupResultNull and long lastValue. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMax.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMax.java index 47911af4521d..fe6d37538392 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMax.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMax.java @@ -45,7 +45,6 @@ public VectorPTFEvaluatorLongMax(WindowFrameDef windowFrameDef, VectorExpression public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Determine maximum of all non-null long column values; maintain isGroupResultNull. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMin.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMin.java index 91d21f2f182d..2867653d10b5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongMin.java @@ -45,7 +45,6 @@ public VectorPTFEvaluatorLongMin(WindowFrameDef windowFrameDef, VectorExpression public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Determine minimum of all non-null long column values; maintain isGroupResultNull. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongSum.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongSum.java index d37bf985dc1f..5318407d6bb2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongSum.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorLongSum.java @@ -42,7 +42,6 @@ public VectorPTFEvaluatorLongSum(WindowFrameDef windowFrameDef, VectorExpression public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Sum all non-null long column values; maintain isGroupResultNull. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorRowNumber.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorRowNumber.java index 3e4bc3d84bc1..b287639c27a2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorRowNumber.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorRowNumber.java @@ -44,7 +44,6 @@ public VectorPTFEvaluatorRowNumber(WindowFrameDef windowFrameDef, VectorExpressi public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); final int size = batch.size; LongColumnVector longColVector = (LongColumnVector) batch.cols[outputColumnNum]; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalAvg.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalAvg.java index e51d1fc5f665..58ad82003c56 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalAvg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalAvg.java @@ -56,7 +56,6 @@ public VectorPTFEvaluatorStreamingDecimalAvg(WindowFrameDef windowFrameDef, Vect public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Sum all non-null decimal column values for avg; maintain isGroupResultNull; after last row of // last group batch compute the group avg when sum is non-null. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMax.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMax.java index 935724271077..dabfa18b323f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMax.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMax.java @@ -48,7 +48,6 @@ public VectorPTFEvaluatorStreamingDecimalMax(WindowFrameDef windowFrameDef, Vect public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Determine maximum of all non-null decimal column values; maintain isNull. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMin.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMin.java index 51b43d7c622b..a729775b00fa 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalMin.java @@ -48,7 +48,6 @@ public VectorPTFEvaluatorStreamingDecimalMin(WindowFrameDef windowFrameDef, Vect public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Determine minimum of all non-null decimal column values; maintain isNull. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalSum.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalSum.java index bc8620ac37f2..3fb86f9379d3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalSum.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDecimalSum.java @@ -48,7 +48,6 @@ public VectorPTFEvaluatorStreamingDecimalSum(WindowFrameDef windowFrameDef, Vect public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Sum all non-null decimal column values; maintain isGroupResultNull. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleAvg.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleAvg.java index f6c5942c065e..18d521c07c26 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleAvg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleAvg.java @@ -50,7 +50,6 @@ public VectorPTFEvaluatorStreamingDoubleAvg(WindowFrameDef windowFrameDef, Vecto public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Sum all non-null double column values for avg; maintain isGroupResultNull; after last row of // last group batch compute the group avg when sum is non-null. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMax.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMax.java index 1d61cc5bf09d..5a40b638e714 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMax.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMax.java @@ -45,7 +45,6 @@ public VectorPTFEvaluatorStreamingDoubleMax(WindowFrameDef windowFrameDef, Vecto public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Determine maximum of all non-null double column values; maintain isNull. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMin.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMin.java index 9ac197d17a8f..14c0b831edd3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleMin.java @@ -45,7 +45,6 @@ public VectorPTFEvaluatorStreamingDoubleMin(WindowFrameDef windowFrameDef, Vecto public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Determine minimum of all non-null double column values; maintain isNull. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleSum.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleSum.java index 8f17663a358d..2e0dfa9a45fb 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleSum.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingDoubleSum.java @@ -45,7 +45,6 @@ public VectorPTFEvaluatorStreamingDoubleSum(WindowFrameDef windowFrameDef, Vecto public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // We do not filter when PTF is in reducer. Preconditions.checkState(!batch.selectedInUse); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongAvg.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongAvg.java index 78d543a0c20d..9117ac16d31a 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongAvg.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongAvg.java @@ -50,7 +50,6 @@ public VectorPTFEvaluatorStreamingLongAvg(WindowFrameDef windowFrameDef, VectorE public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Sum all non-null long column values for avg; maintain isGroupResultNull; after last row of // last group batch compute the group avg when sum is non-null. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMax.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMax.java index 94d19b3376f4..ee2804b5838c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMax.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMax.java @@ -45,7 +45,6 @@ public VectorPTFEvaluatorStreamingLongMax(WindowFrameDef windowFrameDef, VectorE public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Determine maximum of all non-null long column values; maintain isNull. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMin.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMin.java index 2d7caf341524..42ab267e947d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongMin.java @@ -45,7 +45,6 @@ public VectorPTFEvaluatorStreamingLongMin(WindowFrameDef windowFrameDef, VectorE public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Determine minimum of all non-null long column values; maintain isNull. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongSum.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongSum.java index 76bca6b65578..311707966c9e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongSum.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFEvaluatorStreamingLongSum.java @@ -45,7 +45,6 @@ public VectorPTFEvaluatorStreamingLongSum(WindowFrameDef windowFrameDef, VectorE public void evaluateGroupBatch(VectorizedRowBatch batch) throws HiveException { - evaluateInputExpr(batch); // Sum all non-null long column values; maintain isNull. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java index f05683eb3018..bd60d215ff01 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFGroupBatches.java @@ -336,15 +336,6 @@ public Object getValue(int row, int col) throws HiveException { return getValueFromBatch(bufferedBatches.get(rp.batchIndex), col, rp.rowIndexInBatch); } - public Object getValueAndEvaluateInputExpression(VectorPTFEvaluatorBase evaluator, int row, - int col) throws HiveException { - RowPositionInBatch rp = getPosition(row); - BufferedVectorizedRowBatch batch = bufferedBatches.get(rp.batchIndex); - if (!batch.isInputExpressionEvaluated) { - evaluator.evaluateInputExpr(batch); - } - return getValueFromBatch(batch, col, rp.rowIndexInBatch); - } public RowPositionInBatch getPosition(int i) throws HiveException { if (positionCache[i] != null) { @@ -997,7 +988,6 @@ private VectorizedRowBatch createTruncatedBufferedBatch(BufferedVectorizedRowBat partialBatch.size = size; partialBatch.isLastGroupBatch = bufferedBatch.isLastGroupBatch; - partialBatch.isInputExpressionEvaluated = bufferedBatch.isInputExpressionEvaluated; return partialBatch; } @@ -1159,7 +1149,6 @@ private void readBlockFromSpillToMemory(VectorSpillBlock block) throws HiveExcep block.readSingleRowFromBytesContainer(bufferedBatch); bufferedBatch.size += 1; bufferedBatch.isLastGroupBatch = block.isLastGroupBatch[batchIndex]; - bufferedBatch.isInputExpressionEvaluated = block.isInputExpressionEvaluated[batchIndex]; } currentBufferedBatchCount = block.spillBatchCount; partitionMetrics.batchesReadFromSpill += currentBufferedBatchCount; @@ -1187,7 +1176,6 @@ private void spillAndResetCurrentBufferedBatches() throws HiveException { BufferedVectorizedRowBatch bufferedBatch = bufferedBatches.get(i); block.spillBatch(bufferedBatch); block.isLastGroupBatch[i] = bufferedBatch.isLastGroupBatch; - block.isInputExpressionEvaluated[i] = bufferedBatch.isInputExpressionEvaluated; block.spillRowCount += bufferedBatch.size; block.spillBatchCount += 1; if (previousBatch != null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java index 6d03bb5ea5d1..26987a68f442 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorPTFOperator.java @@ -28,7 +28,6 @@ import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation; import org.apache.hadoop.hive.common.type.HiveIntervalDayTime; import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.Utilities; @@ -389,6 +388,18 @@ public void process(Object row, int tag) throws HiveException { } } + /* + Evaluate input expressions on the original batch before it gets buffered. + This ensures that complex expressions read from the correct columns in + the original batch and write their results to scratch columns, which + are then buffered. + */ + for (VectorPTFEvaluatorBase evaluator : evaluators) { + if (evaluator.inputVecExpr != null) { + evaluator.inputVecExpr.evaluate(batch); + } + } + if (isFirstPartition) { isFirstPartition = false; setCurrentPartition(batch); @@ -476,39 +487,27 @@ private void initBufferedColumns() { } private void initExpressionColumns() { - for (int i = 0; i < evaluators.length; i++) { - VectorPTFEvaluatorBase evaluator = evaluators[i]; + for (VectorPTFEvaluatorBase evaluator : evaluators) { /* - * Non-streaming evaluators work on buffered batches, we need to adapt them. Before PTF - * bounded start vectorization (HIVE-24761), VectorExpression.outputColumnNum was closed and - * VectorExpression.inputColumnNum didn't even exist (even though the vast majority of - * VectorExpression subclasses use at least 1 input column). Since VectorPTFOperator and - * VectorPTFGroupBatches work on modified batches (by not storing all the columns, and having - * ordering columns first), the expressions planned in compile-time won't work with the - * original config (column layout). It would make sense to move this logic to compile time, - * because here in runtime, a very simple mapping (bufferedColumnMap) is used, so it might be - * used. However, vectorized expression compilation affects many layers of code (having - * VectorizationContext as the common scope), and moving the calculation of bufferedColumnMap - * and this override logic to compile-time would create much more complicated behavior there - * (probably involving hacking most of the time, or maybe a great re-design) just because of - * the optimized column layout of the PTF vectorization. + * Non-streaming window function evaluators (like SUM or LEAD) work on packed, buffered batches. + * VectorPTFOperator and VectorPTFGroupBatches significantly modify the batch structure to save memory + * (e.g. by dropping unused columns and moving ordering columns to the front). + * + * Because of this dense packing, the absolute column indices assigned by the compiler (e.g., Index 15) + * are no longer valid for the buffer (where it might now be Index 2). + * Therefore, we must dynamically map the evaluator's inputColumnNum to its new location in the packed array. + * + * NOTE: We ONLY patch the evaluator's input index here. We specifically DO NOT patch + * evaluator.inputVecExpr (the math expressions like A+B) because those are now evaluated eagerly + * on the original batch before any packing occurs, so their absolute indices remain perfectly valid. */ if (!evaluator.streamsResult()) { - evaluator.inputColumnNum = IntStream.range(0, bufferedColumnMap.length) - .filter(j -> bufferedColumnMap[j] == evaluator.inputColumnNum).findFirst() - .orElseGet(() -> evaluator.inputColumnNum); - - if (evaluator.inputVecExpr != null) { - for (int j = 0; j < evaluator.inputVecExpr.inputColumnNum.length; j++){ - final int jj = j; // need a final in stream filter - evaluator.inputVecExpr.inputColumnNum[jj] = IntStream.range(0, bufferedColumnMap.length) - .filter(k -> bufferedColumnMap[k] == evaluator.inputVecExpr.inputColumnNum[jj]) - .findFirst().orElseGet(() -> evaluator.inputVecExpr.inputColumnNum[jj]); - } - evaluator.inputVecExpr.outputColumnNum = IntStream.range(0, bufferedColumnMap.length) - .filter(j -> bufferedColumnMap[j] == evaluator.inputVecExpr.outputColumnNum) - .findFirst().orElseGet(() -> evaluator.inputVecExpr.outputColumnNum); - } + evaluator.inputColumnNum = + IntStream.range(0, bufferedColumnMap.length) + .filter(j -> bufferedColumnMap[j] == evaluator.inputColumnNum) + .findFirst() + .orElseGet(() -> evaluator.inputColumnNum); + evaluator.mapCustomColumns(bufferedColumnMap); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorSpillBlockContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorSpillBlockContainer.java index 8d21892b785f..a5c6170324c0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorSpillBlockContainer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/ptf/VectorSpillBlockContainer.java @@ -139,10 +139,6 @@ class VectorSpillBlock { */ boolean[] isLastGroupBatch; - /** - * Stores the isInputExpressionEvaluated property for buffered/spilled batches. - */ - boolean[] isInputExpressionEvaluated; /** *
* Deserializer needs to know which columns to put the deserialized values into. Basically,
@@ -165,7 +161,6 @@ class VectorSpillBlock {
startBatchIndex = blockIndex * blockSize;
startRowIndex = new int[blockSize];
isLastGroupBatch = new boolean[blockSize];
- isInputExpressionEvaluated = new boolean[blockSize];
}
void releaseRowBytesContainer() {
diff --git a/ql/src/test/queries/clientpositive/vector_ptf_nested_input_expr.q b/ql/src/test/queries/clientpositive/vector_ptf_nested_input_expr.q
new file mode 100644
index 000000000000..823954565e98
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_ptf_nested_input_expr.q
@@ -0,0 +1,56 @@
+set hive.cli.print.header=true;
+set hive.vectorized.execution.enabled=true;
+set hive.vectorized.execution.reduce.enabled=true;
+set hive.vectorized.execution.ptf.enabled=true;
+
+CREATE TABLE repro_ptf_decimal (
+ id STRING,
+ val1 DECIMAL(20,2),
+ val2 BIGINT,
+ val3 SMALLINT
+) STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB');
+
+INSERT INTO repro_ptf_decimal VALUES ('req1', 5.0, 2, 1), ('req1', 10.0, 4, 2), ('req2', 10.0, 5, 2);
+
+EXPLAIN VECTORIZATION DETAIL
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_decimal;
+
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_decimal;
+
+CREATE TABLE repro_ptf_long (
+ id STRING,
+ val1 BIGINT,
+ val2 INT,
+ val3 SMALLINT
+) STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB');
+
+INSERT INTO repro_ptf_long VALUES ('req1', 5, 2, 1), ('req1', 10, 4, 2), ('req2', 10, 5, 2);
+
+EXPLAIN VECTORIZATION DETAIL
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_long;
+
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_long;
diff --git a/ql/src/test/results/clientpositive/llap/vector_ptf_nested_input_expr.q.out b/ql/src/test/results/clientpositive/llap/vector_ptf_nested_input_expr.q.out
new file mode 100644
index 000000000000..cf12f6296495
--- /dev/null
+++ b/ql/src/test/results/clientpositive/llap/vector_ptf_nested_input_expr.q.out
@@ -0,0 +1,434 @@
+PREHOOK: query: CREATE TABLE repro_ptf_decimal (
+ id STRING,
+ val1 DECIMAL(20,2),
+ val2 BIGINT,
+ val3 SMALLINT
+) STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repro_ptf_decimal
+POSTHOOK: query: CREATE TABLE repro_ptf_decimal (
+ id STRING,
+ val1 DECIMAL(20,2),
+ val2 BIGINT,
+ val3 SMALLINT
+) STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repro_ptf_decimal
+PREHOOK: query: INSERT INTO repro_ptf_decimal VALUES ('req1', 5.0, 2, 1), ('req1', 10.0, 4, 2), ('req2', 10.0, 5, 2)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@repro_ptf_decimal
+POSTHOOK: query: INSERT INTO repro_ptf_decimal VALUES ('req1', 5.0, 2, 1), ('req1', 10.0, 4, 2), ('req2', 10.0, 5, 2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@repro_ptf_decimal
+POSTHOOK: Lineage: repro_ptf_decimal.id SCRIPT []
+POSTHOOK: Lineage: repro_ptf_decimal.val1 SCRIPT []
+POSTHOOK: Lineage: repro_ptf_decimal.val2 SCRIPT []
+POSTHOOK: Lineage: repro_ptf_decimal.val3 SCRIPT []
+_col0 _col1 _col2 _col3
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_decimal
+PREHOOK: type: QUERY
+PREHOOK: Input: default@repro_ptf_decimal
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_decimal
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@repro_ptf_decimal
+#### A masked pattern was here ####
+Explain
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: repro_ptf_decimal
+ Statistics: Num rows: 3 Data size: 636 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:id:string, 1:val1:decimal(20,2), 2:val2:bigint, 3:val3:smallint, 4:ROW__ID:struct, 5:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: 0 (type: int), val1 (type: decimal(20,2)), id (type: string)
+ null sort order: aaz
+ sort order: +-+
+ Map-reduce partition columns: 0 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 6:int, 1:decimal(20,2), 0:string
+ keyExpressions: ConstantVectorExpression(val 0) -> 6:int
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 7:int
+ valueColumns: 2:bigint, 3:smallint
+ Statistics: Num rows: 3 Data size: 636 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: val2 (type: bigint), val3 (type: smallint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ includeColumns: [0, 1, 2, 3]
+ dataColumns: id:string, val1:decimal(20,2), val2:bigint, val3:smallint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: aaz
+ reduceColumnSortOrder: +-+
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 5
+ dataColumns: KEY.reducesinkkey0:int, KEY.reducesinkkey1:decimal(20,2), KEY.reducesinkkey2:string, VALUE._col0:bigint, VALUE._col1:smallint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [decimal(34,2), bigint, decimal(22,2), decimal(19,0), decimal(23,2), decimal(5,0), decimal(24,2)]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey1 (type: decimal(20,2)), VALUE._col0 (type: bigint), VALUE._col1 (type: smallint)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 1, 3, 4]
+ Statistics: Num rows: 3 Data size: 636 Basic stats: COMPLETE Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: decimal(20,2), _col2: bigint, _col3: smallint
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col1 DESC NULLS FIRST, _col0 ASC NULLS LAST
+ partition by: 0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: sum_window_0
+ arguments: (((_col1 * CAST( 2 AS decimal(10,0))) + CAST( _col2 AS decimal(19,0))) - CAST( _col3 AS decimal(5,0)))
+ name: sum
+ window function: GenericUDAFSumHiveDecimal
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorDecimalSum]
+ functionInputExpressions: [DecimalColSubtractDecimalColumn(col 9:decimal(23,2), col 10:decimal(5,0))(children: DecimalColAddDecimalColumn(col 7:decimal(22,2), col 8:decimal(19,0))(children: DecimalColMultiplyDecimalScalar(col 1:decimal(20,2), val 2) -> 7:decimal(22,2), CastLongToDecimal(col 3:bigint) -> 8:decimal(19,0)) -> 9:decimal(23,2), CastLongToDecimal(col 4:smallint) -> 10:decimal(5,0)) -> 11:decimal(24,2)]
+ functionNames: [sum]
+ keyInputColumns: [2, 1]
+ native: true
+ nonKeyInputColumns: [3, 4]
+ orderExpressions: [col 1:decimal(20,2), col 2:string]
+ outputColumns: [5, 2, 1, 3, 4]
+ outputTypes: [decimal(34,2), string, decimal(20,2), bigint, smallint]
+ partitionExpressions: [ConstantVectorExpression(val 0) -> 6:int]
+ streamingColumns: []
+ Statistics: Num rows: 3 Data size: 636 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: decimal(20,2)), _col2 (type: bigint), _col3 (type: smallint), sum_window_0 (type: decimal(34,2))
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 1, 3, 4, 5]
+ Statistics: Num rows: 3 Data size: 972 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 3 Data size: 972 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_decimal
+PREHOOK: type: QUERY
+PREHOOK: Input: default@repro_ptf_decimal
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_decimal
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@repro_ptf_decimal
+#### A masked pattern was here ####
+id val1 val2 val3 final_total
+req1 10.00 4 2 22.00
+req2 10.00 5 2 45.00
+req1 5.00 2 1 56.00
+PREHOOK: query: CREATE TABLE repro_ptf_long (
+ id STRING,
+ val1 BIGINT,
+ val2 INT,
+ val3 SMALLINT
+) STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repro_ptf_long
+POSTHOOK: query: CREATE TABLE repro_ptf_long (
+ id STRING,
+ val1 BIGINT,
+ val2 INT,
+ val3 SMALLINT
+) STORED AS ORC TBLPROPERTIES ('orc.compress'='ZLIB')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repro_ptf_long
+PREHOOK: query: INSERT INTO repro_ptf_long VALUES ('req1', 5, 2, 1), ('req1', 10, 4, 2), ('req2', 10, 5, 2)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+PREHOOK: Output: default@repro_ptf_long
+POSTHOOK: query: INSERT INTO repro_ptf_long VALUES ('req1', 5, 2, 1), ('req1', 10, 4, 2), ('req2', 10, 5, 2)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+POSTHOOK: Output: default@repro_ptf_long
+POSTHOOK: Lineage: repro_ptf_long.id SCRIPT []
+POSTHOOK: Lineage: repro_ptf_long.val1 SCRIPT []
+POSTHOOK: Lineage: repro_ptf_long.val2 SCRIPT []
+POSTHOOK: Lineage: repro_ptf_long.val3 SCRIPT []
+_col0 _col1 _col2 _col3
+PREHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_long
+PREHOOK: type: QUERY
+PREHOOK: Input: default@repro_ptf_long
+#### A masked pattern was here ####
+POSTHOOK: query: EXPLAIN VECTORIZATION DETAIL
+SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_long
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@repro_ptf_long
+#### A masked pattern was here ####
+Explain
+PLAN VECTORIZATION:
+ enabled: true
+ enabledConditionsMet: [hive.vectorized.execution.enabled IS true]
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: repro_ptf_long
+ Statistics: Num rows: 3 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE
+ TableScan Vectorization:
+ native: true
+ vectorizationSchemaColumns: [0:id:string, 1:val1:bigint, 2:val2:int, 3:val3:smallint, 4:ROW__ID:struct, 5:ROW__IS__DELETED:boolean]
+ Reduce Output Operator
+ key expressions: 0 (type: int), val1 (type: bigint), id (type: string)
+ null sort order: aaz
+ sort order: +-+
+ Map-reduce partition columns: 0 (type: int)
+ Reduce Sink Vectorization:
+ className: VectorReduceSinkObjectHashOperator
+ keyColumns: 6:int, 1:bigint, 0:string
+ keyExpressions: ConstantVectorExpression(val 0) -> 6:int
+ native: true
+ nativeConditionsMet: hive.vectorized.execution.reducesink.new.enabled IS true, hive.execution.engine tez IN [tez] IS true, No PTF TopN IS true, No DISTINCT columns IS true, BinarySortableSerDe for keys IS true, LazyBinarySerDe for values IS true
+ partitionColumns: 7:int
+ valueColumns: 2:int, 3:smallint
+ Statistics: Num rows: 3 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE
+ value expressions: val2 (type: int), val3 (type: smallint)
+ Execution mode: vectorized, llap
+ LLAP IO: all inputs
+ Map Vectorization:
+ enabled: true
+ enabledConditionsMet: hive.vectorized.use.vectorized.input.format IS true
+ inputFormatFeatureSupport: [DECIMAL_64]
+ featureSupportInUse: [DECIMAL_64]
+ inputFileFormats: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ allNative: true
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 4
+ includeColumns: [0, 1, 2, 3]
+ dataColumns: id:string, val1:bigint, val2:int, val3:smallint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint]
+ Reducer 2
+ Execution mode: vectorized, llap
+ Reduce Vectorization:
+ enabled: true
+ enableConditionsMet: hive.vectorized.execution.reduce.enabled IS true, hive.execution.engine tez IN [tez] IS true
+ reduceColumnNullOrder: aaz
+ reduceColumnSortOrder: +-+
+ allNative: false
+ usesVectorUDFAdaptor: false
+ vectorized: true
+ rowBatchContext:
+ dataColumnCount: 5
+ dataColumns: KEY.reducesinkkey0:int, KEY.reducesinkkey1:bigint, KEY.reducesinkkey2:string, VALUE._col0:int, VALUE._col1:smallint
+ partitionColumnCount: 0
+ scratchColumnTypeNames: [bigint, bigint, bigint, bigint, bigint]
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey2 (type: string), KEY.reducesinkkey1 (type: bigint), VALUE._col0 (type: int), VALUE._col1 (type: smallint)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 1, 3, 4]
+ Statistics: Num rows: 3 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE
+ PTF Operator
+ Function definitions:
+ Input definition
+ input alias: ptf_0
+ output shape: _col0: string, _col1: bigint, _col2: int, _col3: smallint
+ type: WINDOWING
+ Windowing table definition
+ input alias: ptf_1
+ name: windowingtablefunction
+ order by: _col1 DESC NULLS FIRST, _col0 ASC NULLS LAST
+ partition by: 0
+ raw input shape:
+ window functions:
+ window function definition
+ alias: sum_window_0
+ arguments: (((_col1 * UDFToLong(2)) + UDFToLong(_col2)) - UDFToLong(_col3))
+ name: sum
+ window function: GenericUDAFSumLong
+ window frame: RANGE PRECEDING(MAX)~CURRENT
+ PTF Vectorization:
+ allEvaluatorsAreStreaming: false
+ className: VectorPTFOperator
+ evaluatorClasses: [VectorPTFEvaluatorLongSum]
+ functionInputExpressions: [LongColSubtractLongColumn(col 8:bigint, col 4:bigint)(children: LongColAddLongColumn(col 7:bigint, col 3:bigint)(children: LongColMultiplyLongScalar(col 1:bigint, val 2) -> 7:bigint, col 3:int) -> 8:bigint, col 4:smallint) -> 9:bigint]
+ functionNames: [sum]
+ keyInputColumns: [2, 1]
+ native: true
+ nonKeyInputColumns: [3, 4]
+ orderExpressions: [col 1:bigint, col 2:string]
+ outputColumns: [5, 2, 1, 3, 4]
+ outputTypes: [bigint, string, bigint, int, smallint]
+ partitionExpressions: [ConstantVectorExpression(val 0) -> 6:int]
+ streamingColumns: []
+ Statistics: Num rows: 3 Data size: 312 Basic stats: COMPLETE Column stats: COMPLETE
+ Select Operator
+ expressions: _col0 (type: string), _col1 (type: bigint), _col2 (type: int), _col3 (type: smallint), sum_window_0 (type: bigint)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4
+ Select Vectorization:
+ className: VectorSelectOperator
+ native: true
+ projectedOutputColumnNums: [2, 1, 3, 4, 5]
+ Statistics: Num rows: 3 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE
+ File Output Operator
+ compressed: false
+ File Sink Vectorization:
+ className: VectorFileSinkOperator
+ native: false
+ Statistics: Num rows: 3 Data size: 336 Basic stats: COMPLETE Column stats: COMPLETE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_long
+PREHOOK: type: QUERY
+PREHOOK: Input: default@repro_ptf_long
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT
+ id,
+ val1,
+ val2,
+ val3,
+ SUM(val1 * 2 + val2 - val3) OVER (ORDER BY val1 DESC, id ASC) AS final_total
+FROM repro_ptf_long
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@repro_ptf_long
+#### A masked pattern was here ####
+id val1 val2 val3 final_total
+req1 10 4 2 22
+req2 10 5 2 45
+req1 5 2 1 56