Skip to content

HIVE-26368: DISTINCT keyword from Count UDF is removed from query pla… #6582

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
soumyakanti3578 wants to merge 1 commit into
base: master
Choose a base branch
from
+359 −6
Open

HIVE-26368: DISTINCT keyword from Count UDF is removed from query pla… #6582

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions ...a/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveExpandDistinctAggregatesRule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,14 @@ public void onMatch(RelOptRuleCall call) {
// arguments then we can use a more efficient form.
final RelMetadataQuery mq = call.getMetadataQuery();
if ((nonDistinctCount == 0) && (argListSets.size() == 1)) {
for (Integer arg : argListSets.iterator().next()) {
List<Integer> argList = argListSets.iterator().next();
// When COUNT(DISTINCT a, b, ...) has multiple columns convertMonopole -> rewriteAggCalls
// would produce COUNT(a, b) without DISTINCT at the top level, which is semantically
// wrong and is rejected by GenericUDAFCount.
if (argList.size() > 1 && numCountDistinct > 0) {
return;
}
for (Integer arg : argList) {
Set<RelColumnOrigin> colOrigs = mq.getColumnOrigins(aggregate.getInput(), arg);
if (null != colOrigs) {
for (RelColumnOrigin colOrig : colOrigs) {
Expand All @@ -173,12 +180,8 @@ public void onMatch(RelOptRuleCall call) {
}
}
}
RelNode converted =
convertMonopole(
aggregate,
argListSets.iterator().next());
RelNode converted = convertMonopole(aggregate, argList);
call.transformTo(converted);
return;
}
}

Expand Down
80 changes: 80 additions & 0 deletions ql/src/test/queries/clientpositive/count_distinct_multi_col.q
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
create table t (col1 string, col2 int, col3 int, col4 date);

insert into t values
('a', 1, 2, '2022-01-01'),
('a', 1, 2, '2022-01-01'),
('a', 1, 2, '2022-01-02'),
('a', 1, 22, '2022-01-02'),
('a', 11, 2, '2022-01-01'),
('a', 11, 2, '2022-01-02');

-- single-column: HiveExpandDistinctAggregatesRule matched -> two HiveAggregate operators in plan
explain cbo
SELECT col1, col2, COUNT(DISTINCT col3) AS cnt
FROM t
GROUP BY col1, col2;

SELECT col1, col2, COUNT(DISTINCT col3) AS cnt
FROM t
GROUP BY col1, col2
ORDER BY col1, col2;

-- multi-column: HiveExpandDistinctAggregatesRule !matched -> single HiveAggregate operator
explain cbo
SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt
FROM t
GROUP BY col1, col2;

SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt
FROM t
GROUP BY col1, col2
ORDER BY col1, col2;

-- 3-column: HiveExpandDistinctAggregatesRule !matched -> single HiveAggregate operator
explain cbo
SELECT col1, COUNT(DISTINCT col2, col3, col4) AS cnt
FROM t
GROUP BY col1;

SELECT col1, COUNT(DISTINCT col2, col3, col4) AS cnt
FROM t
GROUP BY col1;

-- multi-column distinct alongside a non-distinct aggregate: single HiveAggregate operator
explain cbo
SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt, SUM(col3) AS s
FROM t
GROUP BY col1, col2;

SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt, SUM(col3) AS s
FROM t
GROUP BY col1, col2
ORDER BY col1, col2;

-- multiple COUNT DISTINCT with different single-column arg: uses grouping-sets
explain cbo
SELECT col1, COUNT(DISTINCT col3) AS c3, COUNT(DISTINCT col2) AS c2
FROM t
GROUP BY col1;

SELECT col1, COUNT(DISTINCT col3) AS c3, COUNT(DISTINCT col2) AS c2
FROM t
GROUP BY col1;

-- 2 COUNT DISTINCTs each with multiple columns: uses grouping-sets path
explain cbo
SELECT col1, COUNT(DISTINCT col2, col3) AS c1, COUNT(DISTINCT col3, col4) AS c2
FROM t
GROUP BY col1;

SELECT col1, COUNT(DISTINCT col2, col3) AS c1, COUNT(DISTINCT col3, col4) AS c2
FROM t
GROUP BY col1;

-- CTAS
create table t_ctas as
SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt
FROM t
GROUP BY col1, col2;

select * from t_ctas ORDER BY col1, col2;
270 changes: 270 additions & 0 deletions ql/src/test/results/clientpositive/llap/count_distinct_multi_col.q.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,270 @@
PREHOOK: query: create table t (col1 string, col2 int, col3 int, col4 date)
PREHOOK: type: CREATETABLE
PREHOOK: Output: database:default
PREHOOK: Output: default@t
POSTHOOK: query: create table t (col1 string, col2 int, col3 int, col4 date)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: database:default
POSTHOOK: Output: default@t
PREHOOK: query: insert into t values
('a', 1, 2, '2022-01-01'),
('a', 1, 2, '2022-01-01'),
('a', 1, 2, '2022-01-02'),
('a', 1, 22, '2022-01-02'),
('a', 11, 2, '2022-01-01'),
('a', 11, 2, '2022-01-02')
PREHOOK: type: QUERY
PREHOOK: Input: _dummy_database@_dummy_table
PREHOOK: Output: default@t
POSTHOOK: query: insert into t values
('a', 1, 2, '2022-01-01'),
('a', 1, 2, '2022-01-01'),
('a', 1, 2, '2022-01-02'),
('a', 1, 22, '2022-01-02'),
('a', 11, 2, '2022-01-01'),
('a', 11, 2, '2022-01-02')
POSTHOOK: type: QUERY
POSTHOOK: Input: _dummy_database@_dummy_table
POSTHOOK: Output: default@t
POSTHOOK: Lineage: t.col1 SCRIPT []
POSTHOOK: Lineage: t.col2 SCRIPT []
POSTHOOK: Lineage: t.col3 SCRIPT []
POSTHOOK: Lineage: t.col4 SCRIPT []
PREHOOK: query: explain cbo
SELECT col1, col2, COUNT(DISTINCT col3) AS cnt
FROM t
GROUP BY col1, col2
PREHOOK: type: QUERY
PREHOOK: Input: default@t
#### A masked pattern was here ####
POSTHOOK: query: explain cbo
SELECT col1, col2, COUNT(DISTINCT col3) AS cnt
FROM t
GROUP BY col1, col2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
#### A masked pattern was here ####
CBO PLAN:
HiveProject(col1=[$0], col2=[$1], cnt=[$2])
HiveAggregate(group=[{0, 1}], agg#0=[count($2)])
HiveProject(col1=[$0], col2=[$1], col3=[$2])
HiveAggregate(group=[{0, 1, 2}])
HiveTableScan(table=[[default, t]], table:alias=[t])

PREHOOK: query: SELECT col1, col2, COUNT(DISTINCT col3) AS cnt
FROM t
GROUP BY col1, col2
ORDER BY col1, col2
PREHOOK: type: QUERY
PREHOOK: Input: default@t
#### A masked pattern was here ####
POSTHOOK: query: SELECT col1, col2, COUNT(DISTINCT col3) AS cnt
FROM t
GROUP BY col1, col2
ORDER BY col1, col2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
#### A masked pattern was here ####
a 1 2
a 11 1
PREHOOK: query: explain cbo
SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt
FROM t
GROUP BY col1, col2
PREHOOK: type: QUERY
PREHOOK: Input: default@t
#### A masked pattern was here ####
POSTHOOK: query: explain cbo
SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt
FROM t
GROUP BY col1, col2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
#### A masked pattern was here ####
CBO PLAN:
HiveProject(col1=[$0], col2=[$1], cnt=[$2])
HiveAggregate(group=[{0, 1}], agg#0=[count(DISTINCT $2, $3)])
HiveTableScan(table=[[default, t]], table:alias=[t])

PREHOOK: query: SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt
FROM t
GROUP BY col1, col2
ORDER BY col1, col2
PREHOOK: type: QUERY
PREHOOK: Input: default@t
#### A masked pattern was here ####
POSTHOOK: query: SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt
FROM t
GROUP BY col1, col2
ORDER BY col1, col2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
#### A masked pattern was here ####
a 1 3
a 11 2
PREHOOK: query: explain cbo
SELECT col1, COUNT(DISTINCT col2, col3, col4) AS cnt
FROM t
GROUP BY col1
PREHOOK: type: QUERY
PREHOOK: Input: default@t
#### A masked pattern was here ####
POSTHOOK: query: explain cbo
SELECT col1, COUNT(DISTINCT col2, col3, col4) AS cnt
FROM t
GROUP BY col1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
#### A masked pattern was here ####
CBO PLAN:
HiveProject(col1=[$0], cnt=[$1])
HiveAggregate(group=[{0}], agg#0=[count(DISTINCT $1, $2, $3)])
HiveTableScan(table=[[default, t]], table:alias=[t])

PREHOOK: query: SELECT col1, COUNT(DISTINCT col2, col3, col4) AS cnt
FROM t
GROUP BY col1
PREHOOK: type: QUERY
PREHOOK: Input: default@t
#### A masked pattern was here ####
POSTHOOK: query: SELECT col1, COUNT(DISTINCT col2, col3, col4) AS cnt
FROM t
GROUP BY col1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
#### A masked pattern was here ####
a 5
PREHOOK: query: explain cbo
SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt, SUM(col3) AS s
FROM t
GROUP BY col1, col2
PREHOOK: type: QUERY
PREHOOK: Input: default@t
#### A masked pattern was here ####
POSTHOOK: query: explain cbo
SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt, SUM(col3) AS s
FROM t
GROUP BY col1, col2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
#### A masked pattern was here ####
CBO PLAN:
HiveProject(col1=[$0], col2=[$1], cnt=[$2], s=[$3])
HiveAggregate(group=[{0, 1}], agg#0=[count(DISTINCT $2, $3)], agg#1=[sum($2)])
HiveTableScan(table=[[default, t]], table:alias=[t])

PREHOOK: query: SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt, SUM(col3) AS s
FROM t
GROUP BY col1, col2
ORDER BY col1, col2
PREHOOK: type: QUERY
PREHOOK: Input: default@t
#### A masked pattern was here ####
POSTHOOK: query: SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt, SUM(col3) AS s
FROM t
GROUP BY col1, col2
ORDER BY col1, col2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
#### A masked pattern was here ####
a 1 3 28
a 11 2 4
PREHOOK: query: explain cbo
SELECT col1, COUNT(DISTINCT col3) AS c3, COUNT(DISTINCT col2) AS c2
FROM t
GROUP BY col1
PREHOOK: type: QUERY
PREHOOK: Input: default@t
#### A masked pattern was here ####
POSTHOOK: query: explain cbo
SELECT col1, COUNT(DISTINCT col3) AS c3, COUNT(DISTINCT col2) AS c2
FROM t
GROUP BY col1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
#### A masked pattern was here ####
CBO PLAN:
HiveProject(col1=[$0], c3=[$1], c2=[$2])
HiveAggregate(group=[{2}], agg#0=[count($0)], agg#1=[count($1)])
HiveProject($f0=[CASE(AND(=($3, 1), IS NOT NULL($1)), 1, null:INTEGER)], $f1=[CASE(AND(=($3, 2), IS NOT NULL($2)), 1, null:INTEGER)], $f2=[$0])
HiveAggregate(group=[{0, 1, 2}], groups=[[{0, 1}, {0, 2}]], GROUPING__ID=[GROUPING__ID()])
HiveProject($f0=[$0], $f1=[$2], $f2=[$1])
HiveTableScan(table=[[default, t]], table:alias=[t])

PREHOOK: query: SELECT col1, COUNT(DISTINCT col3) AS c3, COUNT(DISTINCT col2) AS c2
FROM t
GROUP BY col1
PREHOOK: type: QUERY
PREHOOK: Input: default@t
#### A masked pattern was here ####
POSTHOOK: query: SELECT col1, COUNT(DISTINCT col3) AS c3, COUNT(DISTINCT col2) AS c2
FROM t
GROUP BY col1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
#### A masked pattern was here ####
a 2 2
PREHOOK: query: explain cbo
SELECT col1, COUNT(DISTINCT col2, col3) AS c1, COUNT(DISTINCT col3, col4) AS c2
FROM t
GROUP BY col1
PREHOOK: type: QUERY
PREHOOK: Input: default@t
#### A masked pattern was here ####
POSTHOOK: query: explain cbo
SELECT col1, COUNT(DISTINCT col2, col3) AS c1, COUNT(DISTINCT col3, col4) AS c2
FROM t
GROUP BY col1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
#### A masked pattern was here ####
CBO PLAN:
HiveProject(col1=[$0], c1=[$1], c2=[$2])
HiveAggregate(group=[{2}], agg#0=[count($0)], agg#1=[count($1)])
HiveProject($f0=[CASE(=($4, 1), 1, null:INTEGER)], $f1=[CASE(=($4, 4), 1, null:INTEGER)], $f2=[$0])
HiveAggregate(group=[{0, 1, 2, 3}], groups=[[{0, 1, 2}, {0, 2, 3}]], GROUPING__ID=[GROUPING__ID()])
HiveProject($f0=[$0], $f1=[$1], $f2=[$2], $f3=[$3])
HiveTableScan(table=[[default, t]], table:alias=[t])

PREHOOK: query: SELECT col1, COUNT(DISTINCT col2, col3) AS c1, COUNT(DISTINCT col3, col4) AS c2
FROM t
GROUP BY col1
PREHOOK: type: QUERY
PREHOOK: Input: default@t
#### A masked pattern was here ####
POSTHOOK: query: SELECT col1, COUNT(DISTINCT col2, col3) AS c1, COUNT(DISTINCT col3, col4) AS c2
FROM t
GROUP BY col1
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t
#### A masked pattern was here ####
a 3 3
PREHOOK: query: create table t_ctas as
SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt
FROM t
GROUP BY col1, col2
PREHOOK: type: CREATETABLE_AS_SELECT
PREHOOK: Input: default@t
PREHOOK: Output: database:default
PREHOOK: Output: default@t_ctas
POSTHOOK: query: create table t_ctas as
SELECT col1, col2, COUNT(DISTINCT col3, col4) AS cnt
FROM t
GROUP BY col1, col2
POSTHOOK: type: CREATETABLE_AS_SELECT
POSTHOOK: Input: default@t
POSTHOOK: Output: database:default
POSTHOOK: Output: default@t_ctas
POSTHOOK: Lineage: t_ctas.cnt EXPRESSION [(t)t.FieldSchema(name:col3, type:int, comment:null), (t)t.FieldSchema(name:col4, type:date, comment:null), ]
POSTHOOK: Lineage: t_ctas.col1 SIMPLE [(t)t.FieldSchema(name:col1, type:string, comment:null), ]
POSTHOOK: Lineage: t_ctas.col2 SIMPLE [(t)t.FieldSchema(name:col2, type:int, comment:null), ]
PREHOOK: query: select * from t_ctas ORDER BY col1, col2
PREHOOK: type: QUERY
PREHOOK: Input: default@t_ctas
#### A masked pattern was here ####
POSTHOOK: query: select * from t_ctas ORDER BY col1, col2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@t_ctas
#### A masked pattern was here ####
a 1 3
a 11 2
Loading