diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java index 3a46498298b6..6dc4ccbd48cf 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java @@ -161,7 +161,14 @@ public void onMatch(RelOptRuleCall call) { // arguments then we can use a more efficient form. final RelMetadataQuery mq = call.getMetadataQuery(); if ((nonDistinctCount == 0) && (argListSets.size() == 1)) { - for (Integer arg : argListSets.iterator().next()) { + List argList = argListSets.iterator().next(); + // When COUNT(DISTINCT a, b, ...) has multiple columns convertMonopole -> rewriteAggCalls + // would produce COUNT(a, b) without DISTINCT at the top level, which is semantically + // wrong and is rejected by GenericUDAFCount. + if (argList.size() > 1 && numCountDistinct > 0) { + return; + } + for (Integer arg : argList) { Set colOrigs = mq.getColumnOrigins(aggregate.getInput(), arg); if (null != colOrigs) { for (RelColumnOrigin colOrig : colOrigs) { @@ -173,12 +180,8 @@ public void onMatch(RelOptRuleCall call) { } } } - RelNode converted = - convertMonopole( - aggregate, - argListSets.iterator().next()); + RelNode converted = convertMonopole(aggregate, argList); call.transformTo(converted); - return; } } diff --git a/ql/src/test/queries/clientpositive/count_distinct_multi_col.q b/ql/src/test/queries/clientpositive/count_distinct_multi_col.q new file mode 100644 index 000000000000..4cb19313a79c --- /dev/null +++ b/ql/src/test/queries/clientpositive/count_distinct_multi_col.q @@ -0,0 +1,80 @@ +create table t (col1 string, col2 int, col3 int, col4 date); + +insert into t values + ('a', 1, 2, '2022-01-01'), + ('a', 1, 2, '2022-01-01'), + ('a', 1, 2, '2022-01-02'), + ('a', 1, 22, '2022-01-02'), + ('a', 11, 2, '2022-01-01'), + ('a', 11, 2, '2022-01-02'); + +-- single-column: HiveExpandDistinctAggregatesRule matched -> two HiveAggregate operators in plan +explain cbo +SELECT col1, col2, COUNT(DISTINCT col3) AS cnt +FROM t +GROUP BY col1, col2; + +SELECT col1, col2, COUNT(DISTINCT col3) AS cnt +FROM t +GROUP BY col1, col2 +ORDER BY col1, col2; + +-- multi-column: HiveExpandDistinctAggregatesRule !matched -> single HiveAggregate operator +explain cbo +SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt +FROM t +GROUP BY col1, col2; + +SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt +FROM t +GROUP BY col1, col2 +ORDER BY col1, col2; + +-- 3-column: HiveExpandDistinctAggregatesRule !matched -> single HiveAggregate operator +explain cbo +SELECT col1, COUNT(DISTINCT col2, col3, col4) AS cnt +FROM t +GROUP BY col1; + +SELECT col1, COUNT(DISTINCT col2, col3, col4) AS cnt +FROM t +GROUP BY col1; + +-- multi-column distinct alongside a non-distinct aggregate: single HiveAggregate operator +explain cbo +SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt, SUM(col3) AS s +FROM t +GROUP BY col1, col2; + +SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt, SUM(col3) AS s +FROM t +GROUP BY col1, col2 +ORDER BY col1, col2; + +-- multiple COUNT DISTINCT with different single-column arg: uses grouping-sets +explain cbo +SELECT col1, COUNT(DISTINCT col3) AS c3, COUNT(DISTINCT col2) AS c2 +FROM t +GROUP BY col1; + +SELECT col1, COUNT(DISTINCT col3) AS c3, COUNT(DISTINCT col2) AS c2 +FROM t +GROUP BY col1; + +-- 2 COUNT DISTINCTs each with multiple columns: uses grouping-sets path +explain cbo +SELECT col1, COUNT(DISTINCT col2, col3) AS c1, COUNT(DISTINCT col3, col4) AS c2 +FROM t +GROUP BY col1; + +SELECT col1, COUNT(DISTINCT col2, col3) AS c1, COUNT(DISTINCT col3, col4) AS c2 +FROM t +GROUP BY col1; + +-- CTAS +create table t_ctas as +SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt +FROM t +GROUP BY col1, col2; + +select * from t_ctas ORDER BY col1, col2; diff --git a/ql/src/test/results/clientpositive/llap/count_distinct_multi_col.q.out b/ql/src/test/results/clientpositive/llap/count_distinct_multi_col.q.out new file mode 100644 index 000000000000..0bcef57e2304 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/count_distinct_multi_col.q.out @@ -0,0 +1,270 @@ +PREHOOK: query: create table t (col1 string, col2 int, col3 int, col4 date) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@t +POSTHOOK: query: create table t (col1 string, col2 int, col3 int, col4 date) +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t +PREHOOK: query: insert into t values + ('a', 1, 2, '2022-01-01'), + ('a', 1, 2, '2022-01-01'), + ('a', 1, 2, '2022-01-02'), + ('a', 1, 22, '2022-01-02'), + ('a', 11, 2, '2022-01-01'), + ('a', 11, 2, '2022-01-02') +PREHOOK: type: QUERY +PREHOOK: Input: _dummy_database@_dummy_table +PREHOOK: Output: default@t +POSTHOOK: query: insert into t values + ('a', 1, 2, '2022-01-01'), + ('a', 1, 2, '2022-01-01'), + ('a', 1, 2, '2022-01-02'), + ('a', 1, 22, '2022-01-02'), + ('a', 11, 2, '2022-01-01'), + ('a', 11, 2, '2022-01-02') +POSTHOOK: type: QUERY +POSTHOOK: Input: _dummy_database@_dummy_table +POSTHOOK: Output: default@t +POSTHOOK: Lineage: t.col1 SCRIPT [] +POSTHOOK: Lineage: t.col2 SCRIPT [] +POSTHOOK: Lineage: t.col3 SCRIPT [] +POSTHOOK: Lineage: t.col4 SCRIPT [] +PREHOOK: query: explain cbo +SELECT col1, col2, COUNT(DISTINCT col3) AS cnt +FROM t +GROUP BY col1, col2 +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: explain cbo +SELECT col1, col2, COUNT(DISTINCT col3) AS cnt +FROM t +GROUP BY col1, col2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +CBO PLAN: +HiveProject(col1=[$0], col2=[$1], cnt=[$2]) + HiveAggregate(group=[{0, 1}], agg#0=[count($2)]) + HiveProject(col1=[$0], col2=[$1], col3=[$2]) + HiveAggregate(group=[{0, 1, 2}]) + HiveTableScan(table=[[default, t]], table:alias=[t]) + +PREHOOK: query: SELECT col1, col2, COUNT(DISTINCT col3) AS cnt +FROM t +GROUP BY col1, col2 +ORDER BY col1, col2 +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: SELECT col1, col2, COUNT(DISTINCT col3) AS cnt +FROM t +GROUP BY col1, col2 +ORDER BY col1, col2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +a 1 2 +a 11 1 +PREHOOK: query: explain cbo +SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt +FROM t +GROUP BY col1, col2 +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: explain cbo +SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt +FROM t +GROUP BY col1, col2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +CBO PLAN: +HiveProject(col1=[$0], col2=[$1], cnt=[$2]) + HiveAggregate(group=[{0, 1}], agg#0=[count(DISTINCT $2, $3)]) + HiveTableScan(table=[[default, t]], table:alias=[t]) + +PREHOOK: query: SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt +FROM t +GROUP BY col1, col2 +ORDER BY col1, col2 +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt +FROM t +GROUP BY col1, col2 +ORDER BY col1, col2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +a 1 3 +a 11 2 +PREHOOK: query: explain cbo +SELECT col1, COUNT(DISTINCT col2, col3, col4) AS cnt +FROM t +GROUP BY col1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: explain cbo +SELECT col1, COUNT(DISTINCT col2, col3, col4) AS cnt +FROM t +GROUP BY col1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +CBO PLAN: +HiveProject(col1=[$0], cnt=[$1]) + HiveAggregate(group=[{0}], agg#0=[count(DISTINCT $1, $2, $3)]) + HiveTableScan(table=[[default, t]], table:alias=[t]) + +PREHOOK: query: SELECT col1, COUNT(DISTINCT col2, col3, col4) AS cnt +FROM t +GROUP BY col1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: SELECT col1, COUNT(DISTINCT col2, col3, col4) AS cnt +FROM t +GROUP BY col1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +a 5 +PREHOOK: query: explain cbo +SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt, SUM(col3) AS s +FROM t +GROUP BY col1, col2 +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: explain cbo +SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt, SUM(col3) AS s +FROM t +GROUP BY col1, col2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +CBO PLAN: +HiveProject(col1=[$0], col2=[$1], cnt=[$2], s=[$3]) + HiveAggregate(group=[{0, 1}], agg#0=[count(DISTINCT $2, $3)], agg#1=[sum($2)]) + HiveTableScan(table=[[default, t]], table:alias=[t]) + +PREHOOK: query: SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt, SUM(col3) AS s +FROM t +GROUP BY col1, col2 +ORDER BY col1, col2 +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt, SUM(col3) AS s +FROM t +GROUP BY col1, col2 +ORDER BY col1, col2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +a 1 3 28 +a 11 2 4 +PREHOOK: query: explain cbo +SELECT col1, COUNT(DISTINCT col3) AS c3, COUNT(DISTINCT col2) AS c2 +FROM t +GROUP BY col1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: explain cbo +SELECT col1, COUNT(DISTINCT col3) AS c3, COUNT(DISTINCT col2) AS c2 +FROM t +GROUP BY col1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +CBO PLAN: +HiveProject(col1=[$0], c3=[$1], c2=[$2]) + HiveAggregate(group=[{2}], agg#0=[count($0)], agg#1=[count($1)]) + HiveProject($f0=[CASE(AND(=($3, 1), IS NOT NULL($1)), 1, null:INTEGER)], $f1=[CASE(AND(=($3, 2), IS NOT NULL($2)), 1, null:INTEGER)], $f2=[$0]) + HiveAggregate(group=[{0, 1, 2}], groups=[[{0, 1}, {0, 2}]], GROUPING__ID=[GROUPING__ID()]) + HiveProject($f0=[$0], $f1=[$2], $f2=[$1]) + HiveTableScan(table=[[default, t]], table:alias=[t]) + +PREHOOK: query: SELECT col1, COUNT(DISTINCT col3) AS c3, COUNT(DISTINCT col2) AS c2 +FROM t +GROUP BY col1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: SELECT col1, COUNT(DISTINCT col3) AS c3, COUNT(DISTINCT col2) AS c2 +FROM t +GROUP BY col1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +a 2 2 +PREHOOK: query: explain cbo +SELECT col1, COUNT(DISTINCT col2, col3) AS c1, COUNT(DISTINCT col3, col4) AS c2 +FROM t +GROUP BY col1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: explain cbo +SELECT col1, COUNT(DISTINCT col2, col3) AS c1, COUNT(DISTINCT col3, col4) AS c2 +FROM t +GROUP BY col1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +CBO PLAN: +HiveProject(col1=[$0], c1=[$1], c2=[$2]) + HiveAggregate(group=[{2}], agg#0=[count($0)], agg#1=[count($1)]) + HiveProject($f0=[CASE(=($4, 1), 1, null:INTEGER)], $f1=[CASE(=($4, 4), 1, null:INTEGER)], $f2=[$0]) + HiveAggregate(group=[{0, 1, 2, 3}], groups=[[{0, 1, 2}, {0, 2, 3}]], GROUPING__ID=[GROUPING__ID()]) + HiveProject($f0=[$0], $f1=[$1], $f2=[$2], $f3=[$3]) + HiveTableScan(table=[[default, t]], table:alias=[t]) + +PREHOOK: query: SELECT col1, COUNT(DISTINCT col2, col3) AS c1, COUNT(DISTINCT col3, col4) AS c2 +FROM t +GROUP BY col1 +PREHOOK: type: QUERY +PREHOOK: Input: default@t +#### A masked pattern was here #### +POSTHOOK: query: SELECT col1, COUNT(DISTINCT col2, col3) AS c1, COUNT(DISTINCT col3, col4) AS c2 +FROM t +GROUP BY col1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t +#### A masked pattern was here #### +a 3 3 +PREHOOK: query: create table t_ctas as +SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt +FROM t +GROUP BY col1, col2 +PREHOOK: type: CREATETABLE_AS_SELECT +PREHOOK: Input: default@t +PREHOOK: Output: database:default +PREHOOK: Output: default@t_ctas +POSTHOOK: query: create table t_ctas as +SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt +FROM t +GROUP BY col1, col2 +POSTHOOK: type: CREATETABLE_AS_SELECT +POSTHOOK: Input: default@t +POSTHOOK: Output: database:default +POSTHOOK: Output: default@t_ctas +POSTHOOK: Lineage: t_ctas.cnt EXPRESSION [(t)t.FieldSchema(name:col3, type:int, comment:null), (t)t.FieldSchema(name:col4, type:date, comment:null), ] +POSTHOOK: Lineage: t_ctas.col1 SIMPLE [(t)t.FieldSchema(name:col1, type:string, comment:null), ] +POSTHOOK: Lineage: t_ctas.col2 SIMPLE [(t)t.FieldSchema(name:col2, type:int, comment:null), ] +PREHOOK: query: select * from t_ctas ORDER BY col1, col2 +PREHOOK: type: QUERY +PREHOOK: Input: default@t_ctas +#### A masked pattern was here #### +POSTHOOK: query: select * from t_ctas ORDER BY col1, col2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@t_ctas +#### A masked pattern was here #### +a 1 3 +a 11 2