From 56f5a4a576475b74821b0ef02b5895c65f4a25c8 Mon Sep 17 00:00:00 2001 From: rtrivedi Date: Mon, 13 Jul 2026 11:20:33 -0500 Subject: [PATCH 1/2] HIVE-29729: AIBOE in HiveFilterProjectTransposeRule during CBO planning for filters over windowing Projects --- .../rules/HiveFilterProjectTransposeRule.java | 12 +- .../cbo_filter_pushdown_windowing_notnull.q | 88 ++++ ...bo_filter_pushdown_windowing_notnull.q.out | 384 ++++++++++++++++++ 3 files changed, 483 insertions(+), 1 deletion(-) create mode 100644 ql/src/test/queries/clientpositive/cbo_filter_pushdown_windowing_notnull.q create mode 100644 ql/src/test/results/clientpositive/llap/cbo_filter_pushdown_windowing_notnull.q.out diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java index 5db95b0ca48f..4311c38cf7f7 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java @@ -338,8 +338,18 @@ public void visit(RelNode node, int ordinal, RelNode parent) { if (node instanceof Filter) { check((Filter) node); } else if (node instanceof Project) { + Project project = (Project) node; + if (RexOver.containsOver(project.getProjects(), null)) { + // RelOptUtil.pushPastProjectUnlessBloat may rewrite RexInputRefs inside RexOver + // window specs using the Project expression list. Window partition/order + // expressions reference the Project input row type, so their indexes may fall + // outside project.getProjects(). This visitor is only a conservative redundancy + // check for IS NOT NULL predicates; if it cannot safely inspect through a + // windowing Project, stop the check instead of failing query planning. + return; + } RexNode condition = HiveRelOptUtil.pushPastProjectUnlessBloat( - filterCondition, (Project) node, bloat); + filterCondition, project, bloat); if (condition != null) { filterCondition = condition; } diff --git a/ql/src/test/queries/clientpositive/cbo_filter_pushdown_windowing_notnull.q b/ql/src/test/queries/clientpositive/cbo_filter_pushdown_windowing_notnull.q new file mode 100644 index 000000000000..5743b5121ba9 --- /dev/null +++ b/ql/src/test/queries/clientpositive/cbo_filter_pushdown_windowing_notnull.q @@ -0,0 +1,88 @@ +-- HIVE-29729: an IS NOT NULL predicate above nested windowing (OVER) Projects must +-- not trip the RedundancyChecker in HiveFilterProjectTransposeRule. +set hive.cbo.fallback.strategy=NEVER; + +create table cl_wl_daily_table ( + user_id string, + lender string, + attributes string, + loan_type string, + whitelisting_status string +) stored as orc; + +explain +select + user_id, + cg_tg_cohort +from ( + select + user_id, + lender, + maximum_principal, + loan_type, + case + when loan_type = 'FRESH' and rnk <= (333333 / user_count) * 0.1 then 'CG' + when loan_type = 'FRESH' and rnk <= (333333 / user_count) then 'TG' + when loan_type = 'REPEAT' and rnk <= (500 / user_count) + and substring(user_id, -2) between '60' and '69' then 'CG' + when loan_type = 'REPEAT' and rnk <= (500 / user_count) + and substring(user_id, -2) between '00' and '45' then 'TG' + when loan_type = 'REPEAT' and rnk <= (500 / user_count) then 'TG-TELE' + end as cg_tg_cohort + from ( + select + user_id, + lender, + maximum_principal, + loan_type, + percent_rank() over(partition by loan_type order by rnk) as rnk, + count(user_id) over(partition by loan_type) as user_count + from ( + select + user_id, + lender, + maximum_principal, + loan_type, + row_number() over(partition by loan_type order by rand()) as rnk + from ( + select + wl.user_id as user_id, + max(lender) as lender, + max(maximum_principal) as maximum_principal, + max(loan_type) as loan_type + from ( + select + user_id, + lender, + case + when repeat_wl = 0 and platform_repeat_wl = 0 then 'FRESH' + else 'REPEAT' + end as loan_type, + cast(maximum_principal / 100 as int) as maximum_principal + from ( + select + user_id, + concat_ws('#', sort_array(collect_set(lender))) as lender, + max( + case + when coalesce(get_json_object(attributes, '$.maximum_principal'), '_') = '_' then 0 + else get_json_object(attributes, '$.maximum_principal') + end + ) as maximum_principal, + max(case when loan_type = 'REPEAT' then 1 else 0 end) as repeat_wl, + max(case when loan_type = 'PLATFORM_REPEAT' then 1 else 0 end) as platform_repeat_wl + from cl_wl_daily_table + where whitelisting_status = 'Currently WL' + group by user_id + ) base + where maximum_principal > 0 + and maximum_principal is not null + and maximum_principal <> '_' + ) wl + group by + wl.user_id + ) grouped_users + ) ranked_once + ) ranked_twice +) cohorted +where cg_tg_cohort is not null; diff --git a/ql/src/test/results/clientpositive/llap/cbo_filter_pushdown_windowing_notnull.q.out b/ql/src/test/results/clientpositive/llap/cbo_filter_pushdown_windowing_notnull.q.out new file mode 100644 index 000000000000..d11b4719a756 --- /dev/null +++ b/ql/src/test/results/clientpositive/llap/cbo_filter_pushdown_windowing_notnull.q.out @@ -0,0 +1,384 @@ +PREHOOK: query: create table cl_wl_daily_table ( + user_id string, + lender string, + attributes string, + loan_type string, + whitelisting_status string +) stored as orc +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@cl_wl_daily_table +POSTHOOK: query: create table cl_wl_daily_table ( + user_id string, + lender string, + attributes string, + loan_type string, + whitelisting_status string +) stored as orc +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@cl_wl_daily_table +PREHOOK: query: explain +select + user_id, + cg_tg_cohort +from ( + select + user_id, + lender, + maximum_principal, + loan_type, + case + when loan_type = 'FRESH' and rnk <= (333333 / user_count) * 0.1 then 'CG' + when loan_type = 'FRESH' and rnk <= (333333 / user_count) then 'TG' + when loan_type = 'REPEAT' and rnk <= (500 / user_count) + and substring(user_id, -2) between '60' and '69' then 'CG' + when loan_type = 'REPEAT' and rnk <= (500 / user_count) + and substring(user_id, -2) between '00' and '45' then 'TG' + when loan_type = 'REPEAT' and rnk <= (500 / user_count) then 'TG-TELE' + end as cg_tg_cohort + from ( + select + user_id, + lender, + maximum_principal, + loan_type, + percent_rank() over(partition by loan_type order by rnk) as rnk, + count(user_id) over(partition by loan_type) as user_count + from ( + select + user_id, + lender, + maximum_principal, + loan_type, + row_number() over(partition by loan_type order by rand()) as rnk + from ( + select + wl.user_id as user_id, + max(lender) as lender, + max(maximum_principal) as maximum_principal, + max(loan_type) as loan_type + from ( + select + user_id, + lender, + case + when repeat_wl = 0 and platform_repeat_wl = 0 then 'FRESH' + else 'REPEAT' + end as loan_type, + cast(maximum_principal / 100 as int) as maximum_principal + from ( + select + user_id, + concat_ws('#', sort_array(collect_set(lender))) as lender, + max( + case + when coalesce(get_json_object(attributes, '$.maximum_principal'), '_') = '_' then 0 + else get_json_object(attributes, '$.maximum_principal') + end + ) as maximum_principal, + max(case when loan_type = 'REPEAT' then 1 else 0 end) as repeat_wl, + max(case when loan_type = 'PLATFORM_REPEAT' then 1 else 0 end) as platform_repeat_wl + from cl_wl_daily_table + where whitelisting_status = 'Currently WL' + group by user_id + ) base + where maximum_principal > 0 + and maximum_principal is not null + and maximum_principal <> '_' + ) wl + group by + wl.user_id + ) grouped_users + ) ranked_once + ) ranked_twice +) cohorted +where cg_tg_cohort is not null +PREHOOK: type: QUERY +PREHOOK: Input: default@cl_wl_daily_table +#### A masked pattern was here #### +POSTHOOK: query: explain +select + user_id, + cg_tg_cohort +from ( + select + user_id, + lender, + maximum_principal, + loan_type, + case + when loan_type = 'FRESH' and rnk <= (333333 / user_count) * 0.1 then 'CG' + when loan_type = 'FRESH' and rnk <= (333333 / user_count) then 'TG' + when loan_type = 'REPEAT' and rnk <= (500 / user_count) + and substring(user_id, -2) between '60' and '69' then 'CG' + when loan_type = 'REPEAT' and rnk <= (500 / user_count) + and substring(user_id, -2) between '00' and '45' then 'TG' + when loan_type = 'REPEAT' and rnk <= (500 / user_count) then 'TG-TELE' + end as cg_tg_cohort + from ( + select + user_id, + lender, + maximum_principal, + loan_type, + percent_rank() over(partition by loan_type order by rnk) as rnk, + count(user_id) over(partition by loan_type) as user_count + from ( + select + user_id, + lender, + maximum_principal, + loan_type, + row_number() over(partition by loan_type order by rand()) as rnk + from ( + select + wl.user_id as user_id, + max(lender) as lender, + max(maximum_principal) as maximum_principal, + max(loan_type) as loan_type + from ( + select + user_id, + lender, + case + when repeat_wl = 0 and platform_repeat_wl = 0 then 'FRESH' + else 'REPEAT' + end as loan_type, + cast(maximum_principal / 100 as int) as maximum_principal + from ( + select + user_id, + concat_ws('#', sort_array(collect_set(lender))) as lender, + max( + case + when coalesce(get_json_object(attributes, '$.maximum_principal'), '_') = '_' then 0 + else get_json_object(attributes, '$.maximum_principal') + end + ) as maximum_principal, + max(case when loan_type = 'REPEAT' then 1 else 0 end) as repeat_wl, + max(case when loan_type = 'PLATFORM_REPEAT' then 1 else 0 end) as platform_repeat_wl + from cl_wl_daily_table + where whitelisting_status = 'Currently WL' + group by user_id + ) base + where maximum_principal > 0 + and maximum_principal is not null + and maximum_principal <> '_' + ) wl + group by + wl.user_id + ) grouped_users + ) ranked_once + ) ranked_twice +) cohorted +where cg_tg_cohort is not null +POSTHOOK: type: QUERY +POSTHOOK: Input: default@cl_wl_daily_table +#### A masked pattern was here #### +STAGE DEPENDENCIES: + Stage-1 is a root stage + Stage-0 depends on stages: Stage-1 + +STAGE PLANS: + Stage: Stage-1 + Tez +#### A masked pattern was here #### + Edges: + Reducer 2 <- Map 1 (SIMPLE_EDGE) + Reducer 3 <- Reducer 2 (SIMPLE_EDGE) + Reducer 4 <- Reducer 3 (SIMPLE_EDGE) + Reducer 5 <- Reducer 4 (SIMPLE_EDGE) +#### A masked pattern was here #### + Vertices: + Map 1 + Map Operator Tree: + TableScan + alias: cl_wl_daily_table + filterExpr: (whitelisting_status = 'Currently WL') (type: boolean) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: (whitelisting_status = 'Currently WL') (type: boolean) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: user_id (type: string), lender (type: string), if(if(get_json_object(attributes, '$.maximum_principal') is not null, (get_json_object(attributes, '$.maximum_principal') = '_'), true), '0', get_json_object(attributes, '$.maximum_principal')) (type: string), if((loan_type = 'REPEAT'), 1, 0) (type: int), if((loan_type = 'PLATFORM_REPEAT'), 1, 0) (type: int) + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: collect_set(_col1), max(_col2), max(_col3), max(_col4) + keys: _col0 (type: string) + minReductionHashAggr: 0.99 + mode: hash + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col0 (type: string) + null sort order: z + sort order: + + Map-reduce partition columns: _col0 (type: string) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + value expressions: _col1 (type: array), _col2 (type: string), _col3 (type: int), _col4 (type: int) + Execution mode: llap + LLAP IO: all inputs + Reducer 2 + Execution mode: llap + Reduce Operator Tree: + Group By Operator + aggregations: collect_set(VALUE._col0), max(VALUE._col1), max(VALUE._col2), max(VALUE._col3) + keys: KEY._col0 (type: string) + mode: mergepartial + outputColumnNames: _col0, _col1, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: int) + outputColumnNames: _col0, _col2, _col3, _col4 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: ((UDFToDouble(_col2) > 0.0D) and (_col2 <> '_')) (type: boolean) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), if(((_col3 = 0) and (_col4 = 0)), 'FRESH', 'REPEAT') (type: string) + outputColumnNames: _col0, _col3 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Group By Operator + aggregations: max(_col3) + keys: _col0 (type: string) + mode: complete + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string), rand() (type: double) + null sort order: az + sort order: ++ + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Reducer 3 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: rand() ASC NULLS LAST + partition by: _col1 + raw input shape: + window functions: + window function definition + alias: row_number_window_0 + name: row_number + window function: GenericUDAFRowNumberEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col0 (type: string), _col1 (type: string), row_number_window_0 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string), _col2 (type: int) + null sort order: az + sort order: ++ + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + value expressions: _col0 (type: string) + Reducer 4 + Execution mode: llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: string, _col1: string, _col2: int + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 ASC NULLS LAST + partition by: _col1 + raw input shape: + window functions: + window function definition + alias: percent_rank_window_0 + arguments: _col2 + name: percent_rank + window function: GenericUDAFPercentRankEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + isPivotResult: true + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: percent_rank_window_0 (type: double), _col0 (type: string), _col1 (type: string) + outputColumnNames: percent_rank_window_0, _col0, _col1 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Reduce Output Operator + key expressions: _col1 (type: string) + null sort order: a + sort order: + + Map-reduce partition columns: _col1 (type: string) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + value expressions: percent_rank_window_0 (type: double), _col0 (type: string) + Reducer 5 + Execution mode: vectorized, llap + Reduce Operator Tree: + Select Operator + expressions: VALUE._col0 (type: double), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string) + outputColumnNames: _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + PTF Operator + Function definitions: + Input definition + input alias: ptf_0 + output shape: _col0: double, _col1: string, _col2: string + type: WINDOWING + Windowing table definition + input alias: ptf_1 + name: windowingtablefunction + order by: _col2 ASC NULLS FIRST + partition by: _col2 + raw input shape: + window functions: + window function definition + alias: count_window_1 + arguments: _col1 + name: count + window function: GenericUDAFCountEvaluator + window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: count_window_1 (type: bigint), _col0 (type: double), _col1 (type: string), _col2 (type: string) + outputColumnNames: count_window_1, _col0, _col1, _col2 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Filter Operator + predicate: CASE WHEN (((_col2 = 'FRESH') and (_col0 <= ((333333.0D / UDFToDouble(count_window_1)) * 0.1D)))) THEN (true) WHEN (((_col2 = 'FRESH') and (_col0 <= (333333.0D / UDFToDouble(count_window_1))))) THEN (true) WHEN (((_col2 = 'REPEAT') and (_col0 <= (500.0D / UDFToDouble(count_window_1))) and substring(_col1, -2) BETWEEN '60' AND '69')) THEN (true) WHEN (((_col2 = 'REPEAT') and (_col0 <= (500.0D / UDFToDouble(count_window_1))) and substring(_col1, -2) BETWEEN '00' AND '45')) THEN (true) WHEN (((_col2 = 'REPEAT') and (_col0 <= (500.0D / UDFToDouble(count_window_1))))) THEN (true) ELSE (false) END (type: boolean) + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + Select Operator + expressions: _col1 (type: string), CASE WHEN (((_col2 = 'FRESH') and (_col0 <= ((333333.0D / UDFToDouble(count_window_1)) * 0.1D)))) THEN ('CG') WHEN (((_col2 = 'FRESH') and (_col0 <= (333333.0D / UDFToDouble(count_window_1))))) THEN ('TG') WHEN (((_col2 = 'REPEAT') and (_col0 <= (500.0D / UDFToDouble(count_window_1))) and substring(_col1, -2) BETWEEN '60' AND '69')) THEN ('CG') WHEN (((_col2 = 'REPEAT') and (_col0 <= (500.0D / UDFToDouble(count_window_1))) and substring(_col1, -2) BETWEEN '00' AND '45')) THEN ('TG') WHEN (((_col2 = 'REPEAT') and (_col0 <= (500.0D / UDFToDouble(count_window_1))))) THEN ('TG-TELE') ELSE (null) END (type: string) + outputColumnNames: _col0, _col1 + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + File Output Operator + compressed: false + Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE + table: + input format: org.apache.hadoop.mapred.SequenceFileInputFormat + output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat + serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe + + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + ListSink + From 9b17b38367e7f7e647177a2e3efd9012c19190a6 Mon Sep 17 00:00:00 2001 From: rtrivedi Date: Tue, 14 Jul 2026 12:32:04 -0500 Subject: [PATCH 2/2] HIVE-29729: Addressed review comments --- .../rules/HiveFilterProjectTransposeRule.java | 19 +- .../cbo_filter_pushdown_windowing_notnull.q | 88 +--- ...bo_filter_pushdown_windowing_notnull.q.out | 391 +----------------- 3 files changed, 34 insertions(+), 464 deletions(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java index 4311c38cf7f7..0f334b335501 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java @@ -337,21 +337,12 @@ public void visit(RelNode node, int ordinal, RelNode parent) { } else { if (node instanceof Filter) { check((Filter) node); - } else if (node instanceof Project) { - Project project = (Project) node; - if (RexOver.containsOver(project.getProjects(), null)) { - // RelOptUtil.pushPastProjectUnlessBloat may rewrite RexInputRefs inside RexOver - // window specs using the Project expression list. Window partition/order - // expressions reference the Project input row type, so their indexes may fall - // outside project.getProjects(). This visitor is only a conservative redundancy - // check for IS NOT NULL predicates; if it cannot safely inspect through a - // windowing Project, stop the check instead of failing query planning. - return; - } - RexNode condition = HiveRelOptUtil.pushPastProjectUnlessBloat( + } else if (node instanceof Project project) { + filterCondition = HiveRelOptUtil.pushPastProjectUnlessBloat( filterCondition, project, bloat); - if (condition != null) { - filterCondition = condition; + if (filterCondition == null) { + // the condition could not be pushed, so bail out + return; } } else { // we do not support other operators for now diff --git a/ql/src/test/queries/clientpositive/cbo_filter_pushdown_windowing_notnull.q b/ql/src/test/queries/clientpositive/cbo_filter_pushdown_windowing_notnull.q index 5743b5121ba9..501cd2a4bba2 100644 --- a/ql/src/test/queries/clientpositive/cbo_filter_pushdown_windowing_notnull.q +++ b/ql/src/test/queries/clientpositive/cbo_filter_pushdown_windowing_notnull.q @@ -2,87 +2,15 @@ -- not trip the RedundancyChecker in HiveFilterProjectTransposeRule. set hive.cbo.fallback.strategy=NEVER; -create table cl_wl_daily_table ( - user_id string, - lender string, - attributes string, - loan_type string, - whitelisting_status string -) stored as orc; +create table tab1 (id string); -explain -select - user_id, - cg_tg_cohort +select rnk2 from ( select - user_id, - lender, - maximum_principal, - loan_type, - case - when loan_type = 'FRESH' and rnk <= (333333 / user_count) * 0.1 then 'CG' - when loan_type = 'FRESH' and rnk <= (333333 / user_count) then 'TG' - when loan_type = 'REPEAT' and rnk <= (500 / user_count) - and substring(user_id, -2) between '60' and '69' then 'CG' - when loan_type = 'REPEAT' and rnk <= (500 / user_count) - and substring(user_id, -2) between '00' and '45' then 'TG' - when loan_type = 'REPEAT' and rnk <= (500 / user_count) then 'TG-TELE' - end as cg_tg_cohort + row_number() over(order by rnk) as rnk2 from ( - select - user_id, - lender, - maximum_principal, - loan_type, - percent_rank() over(partition by loan_type order by rnk) as rnk, - count(user_id) over(partition by loan_type) as user_count - from ( - select - user_id, - lender, - maximum_principal, - loan_type, - row_number() over(partition by loan_type order by rand()) as rnk - from ( - select - wl.user_id as user_id, - max(lender) as lender, - max(maximum_principal) as maximum_principal, - max(loan_type) as loan_type - from ( - select - user_id, - lender, - case - when repeat_wl = 0 and platform_repeat_wl = 0 then 'FRESH' - else 'REPEAT' - end as loan_type, - cast(maximum_principal / 100 as int) as maximum_principal - from ( - select - user_id, - concat_ws('#', sort_array(collect_set(lender))) as lender, - max( - case - when coalesce(get_json_object(attributes, '$.maximum_principal'), '_') = '_' then 0 - else get_json_object(attributes, '$.maximum_principal') - end - ) as maximum_principal, - max(case when loan_type = 'REPEAT' then 1 else 0 end) as repeat_wl, - max(case when loan_type = 'PLATFORM_REPEAT' then 1 else 0 end) as platform_repeat_wl - from cl_wl_daily_table - where whitelisting_status = 'Currently WL' - group by user_id - ) base - where maximum_principal > 0 - and maximum_principal is not null - and maximum_principal <> '_' - ) wl - group by - wl.user_id - ) grouped_users - ) ranked_once - ) ranked_twice -) cohorted -where cg_tg_cohort is not null; + select count(*) over() as rnk + from (select count(*) from tab1) t + ) t2 +) t3 +where rnk2 is not null; diff --git a/ql/src/test/results/clientpositive/llap/cbo_filter_pushdown_windowing_notnull.q.out b/ql/src/test/results/clientpositive/llap/cbo_filter_pushdown_windowing_notnull.q.out index d11b4719a756..fd786ea672b8 100644 --- a/ql/src/test/results/clientpositive/llap/cbo_filter_pushdown_windowing_notnull.q.out +++ b/ql/src/test/results/clientpositive/llap/cbo_filter_pushdown_windowing_notnull.q.out @@ -1,384 +1,35 @@ -PREHOOK: query: create table cl_wl_daily_table ( - user_id string, - lender string, - attributes string, - loan_type string, - whitelisting_status string -) stored as orc +PREHOOK: query: create table tab1 (id string) PREHOOK: type: CREATETABLE PREHOOK: Output: database:default -PREHOOK: Output: default@cl_wl_daily_table -POSTHOOK: query: create table cl_wl_daily_table ( - user_id string, - lender string, - attributes string, - loan_type string, - whitelisting_status string -) stored as orc +PREHOOK: Output: default@tab1 +POSTHOOK: query: create table tab1 (id string) POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:default -POSTHOOK: Output: default@cl_wl_daily_table -PREHOOK: query: explain -select - user_id, - cg_tg_cohort +POSTHOOK: Output: default@tab1 +PREHOOK: query: select rnk2 from ( select - user_id, - lender, - maximum_principal, - loan_type, - case - when loan_type = 'FRESH' and rnk <= (333333 / user_count) * 0.1 then 'CG' - when loan_type = 'FRESH' and rnk <= (333333 / user_count) then 'TG' - when loan_type = 'REPEAT' and rnk <= (500 / user_count) - and substring(user_id, -2) between '60' and '69' then 'CG' - when loan_type = 'REPEAT' and rnk <= (500 / user_count) - and substring(user_id, -2) between '00' and '45' then 'TG' - when loan_type = 'REPEAT' and rnk <= (500 / user_count) then 'TG-TELE' - end as cg_tg_cohort + row_number() over(order by rnk) as rnk2 from ( - select - user_id, - lender, - maximum_principal, - loan_type, - percent_rank() over(partition by loan_type order by rnk) as rnk, - count(user_id) over(partition by loan_type) as user_count - from ( - select - user_id, - lender, - maximum_principal, - loan_type, - row_number() over(partition by loan_type order by rand()) as rnk - from ( - select - wl.user_id as user_id, - max(lender) as lender, - max(maximum_principal) as maximum_principal, - max(loan_type) as loan_type - from ( - select - user_id, - lender, - case - when repeat_wl = 0 and platform_repeat_wl = 0 then 'FRESH' - else 'REPEAT' - end as loan_type, - cast(maximum_principal / 100 as int) as maximum_principal - from ( - select - user_id, - concat_ws('#', sort_array(collect_set(lender))) as lender, - max( - case - when coalesce(get_json_object(attributes, '$.maximum_principal'), '_') = '_' then 0 - else get_json_object(attributes, '$.maximum_principal') - end - ) as maximum_principal, - max(case when loan_type = 'REPEAT' then 1 else 0 end) as repeat_wl, - max(case when loan_type = 'PLATFORM_REPEAT' then 1 else 0 end) as platform_repeat_wl - from cl_wl_daily_table - where whitelisting_status = 'Currently WL' - group by user_id - ) base - where maximum_principal > 0 - and maximum_principal is not null - and maximum_principal <> '_' - ) wl - group by - wl.user_id - ) grouped_users - ) ranked_once - ) ranked_twice -) cohorted -where cg_tg_cohort is not null + select count(*) over() as rnk + from (select count(*) from tab1) t + ) t2 +) t3 +where rnk2 is not null PREHOOK: type: QUERY -PREHOOK: Input: default@cl_wl_daily_table +PREHOOK: Input: default@tab1 #### A masked pattern was here #### -POSTHOOK: query: explain -select - user_id, - cg_tg_cohort +POSTHOOK: query: select rnk2 from ( select - user_id, - lender, - maximum_principal, - loan_type, - case - when loan_type = 'FRESH' and rnk <= (333333 / user_count) * 0.1 then 'CG' - when loan_type = 'FRESH' and rnk <= (333333 / user_count) then 'TG' - when loan_type = 'REPEAT' and rnk <= (500 / user_count) - and substring(user_id, -2) between '60' and '69' then 'CG' - when loan_type = 'REPEAT' and rnk <= (500 / user_count) - and substring(user_id, -2) between '00' and '45' then 'TG' - when loan_type = 'REPEAT' and rnk <= (500 / user_count) then 'TG-TELE' - end as cg_tg_cohort + row_number() over(order by rnk) as rnk2 from ( - select - user_id, - lender, - maximum_principal, - loan_type, - percent_rank() over(partition by loan_type order by rnk) as rnk, - count(user_id) over(partition by loan_type) as user_count - from ( - select - user_id, - lender, - maximum_principal, - loan_type, - row_number() over(partition by loan_type order by rand()) as rnk - from ( - select - wl.user_id as user_id, - max(lender) as lender, - max(maximum_principal) as maximum_principal, - max(loan_type) as loan_type - from ( - select - user_id, - lender, - case - when repeat_wl = 0 and platform_repeat_wl = 0 then 'FRESH' - else 'REPEAT' - end as loan_type, - cast(maximum_principal / 100 as int) as maximum_principal - from ( - select - user_id, - concat_ws('#', sort_array(collect_set(lender))) as lender, - max( - case - when coalesce(get_json_object(attributes, '$.maximum_principal'), '_') = '_' then 0 - else get_json_object(attributes, '$.maximum_principal') - end - ) as maximum_principal, - max(case when loan_type = 'REPEAT' then 1 else 0 end) as repeat_wl, - max(case when loan_type = 'PLATFORM_REPEAT' then 1 else 0 end) as platform_repeat_wl - from cl_wl_daily_table - where whitelisting_status = 'Currently WL' - group by user_id - ) base - where maximum_principal > 0 - and maximum_principal is not null - and maximum_principal <> '_' - ) wl - group by - wl.user_id - ) grouped_users - ) ranked_once - ) ranked_twice -) cohorted -where cg_tg_cohort is not null + select count(*) over() as rnk + from (select count(*) from tab1) t + ) t2 +) t3 +where rnk2 is not null POSTHOOK: type: QUERY -POSTHOOK: Input: default@cl_wl_daily_table +POSTHOOK: Input: default@tab1 #### A masked pattern was here #### -STAGE DEPENDENCIES: - Stage-1 is a root stage - Stage-0 depends on stages: Stage-1 - -STAGE PLANS: - Stage: Stage-1 - Tez -#### A masked pattern was here #### - Edges: - Reducer 2 <- Map 1 (SIMPLE_EDGE) - Reducer 3 <- Reducer 2 (SIMPLE_EDGE) - Reducer 4 <- Reducer 3 (SIMPLE_EDGE) - Reducer 5 <- Reducer 4 (SIMPLE_EDGE) -#### A masked pattern was here #### - Vertices: - Map 1 - Map Operator Tree: - TableScan - alias: cl_wl_daily_table - filterExpr: (whitelisting_status = 'Currently WL') (type: boolean) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: (whitelisting_status = 'Currently WL') (type: boolean) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: user_id (type: string), lender (type: string), if(if(get_json_object(attributes, '$.maximum_principal') is not null, (get_json_object(attributes, '$.maximum_principal') = '_'), true), '0', get_json_object(attributes, '$.maximum_principal')) (type: string), if((loan_type = 'REPEAT'), 1, 0) (type: int), if((loan_type = 'PLATFORM_REPEAT'), 1, 0) (type: int) - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: collect_set(_col1), max(_col2), max(_col3), max(_col4) - keys: _col0 (type: string) - minReductionHashAggr: 0.99 - mode: hash - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col0 (type: string) - null sort order: z - sort order: + - Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - value expressions: _col1 (type: array), _col2 (type: string), _col3 (type: int), _col4 (type: int) - Execution mode: llap - LLAP IO: all inputs - Reducer 2 - Execution mode: llap - Reduce Operator Tree: - Group By Operator - aggregations: collect_set(VALUE._col0), max(VALUE._col1), max(VALUE._col2), max(VALUE._col3) - keys: KEY._col0 (type: string) - mode: mergepartial - outputColumnNames: _col0, _col1, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col2 (type: string), _col3 (type: int), _col4 (type: int) - outputColumnNames: _col0, _col2, _col3, _col4 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: ((UDFToDouble(_col2) > 0.0D) and (_col2 <> '_')) (type: boolean) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), if(((_col3 = 0) and (_col4 = 0)), 'FRESH', 'REPEAT') (type: string) - outputColumnNames: _col0, _col3 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Group By Operator - aggregations: max(_col3) - keys: _col0 (type: string) - mode: complete - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string), rand() (type: double) - null sort order: az - sort order: ++ - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Reducer 3 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: string, _col1: string - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: rand() ASC NULLS LAST - partition by: _col1 - raw input shape: - window functions: - window function definition - alias: row_number_window_0 - name: row_number - window function: GenericUDAFRowNumberEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col0 (type: string), _col1 (type: string), row_number_window_0 (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string), _col2 (type: int) - null sort order: az - sort order: ++ - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - value expressions: _col0 (type: string) - Reducer 4 - Execution mode: llap - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string), KEY.reducesinkkey1 (type: int) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: string, _col1: string, _col2: int - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col2 ASC NULLS LAST - partition by: _col1 - raw input shape: - window functions: - window function definition - alias: percent_rank_window_0 - arguments: _col2 - name: percent_rank - window function: GenericUDAFPercentRankEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - isPivotResult: true - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: percent_rank_window_0 (type: double), _col0 (type: string), _col1 (type: string) - outputColumnNames: percent_rank_window_0, _col0, _col1 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Reduce Output Operator - key expressions: _col1 (type: string) - null sort order: a - sort order: + - Map-reduce partition columns: _col1 (type: string) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - value expressions: percent_rank_window_0 (type: double), _col0 (type: string) - Reducer 5 - Execution mode: vectorized, llap - Reduce Operator Tree: - Select Operator - expressions: VALUE._col0 (type: double), VALUE._col1 (type: string), KEY.reducesinkkey0 (type: string) - outputColumnNames: _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - PTF Operator - Function definitions: - Input definition - input alias: ptf_0 - output shape: _col0: double, _col1: string, _col2: string - type: WINDOWING - Windowing table definition - input alias: ptf_1 - name: windowingtablefunction - order by: _col2 ASC NULLS FIRST - partition by: _col2 - raw input shape: - window functions: - window function definition - alias: count_window_1 - arguments: _col1 - name: count - window function: GenericUDAFCountEvaluator - window frame: ROWS PRECEDING(MAX)~FOLLOWING(MAX) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: count_window_1 (type: bigint), _col0 (type: double), _col1 (type: string), _col2 (type: string) - outputColumnNames: count_window_1, _col0, _col1, _col2 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Filter Operator - predicate: CASE WHEN (((_col2 = 'FRESH') and (_col0 <= ((333333.0D / UDFToDouble(count_window_1)) * 0.1D)))) THEN (true) WHEN (((_col2 = 'FRESH') and (_col0 <= (333333.0D / UDFToDouble(count_window_1))))) THEN (true) WHEN (((_col2 = 'REPEAT') and (_col0 <= (500.0D / UDFToDouble(count_window_1))) and substring(_col1, -2) BETWEEN '60' AND '69')) THEN (true) WHEN (((_col2 = 'REPEAT') and (_col0 <= (500.0D / UDFToDouble(count_window_1))) and substring(_col1, -2) BETWEEN '00' AND '45')) THEN (true) WHEN (((_col2 = 'REPEAT') and (_col0 <= (500.0D / UDFToDouble(count_window_1))))) THEN (true) ELSE (false) END (type: boolean) - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - Select Operator - expressions: _col1 (type: string), CASE WHEN (((_col2 = 'FRESH') and (_col0 <= ((333333.0D / UDFToDouble(count_window_1)) * 0.1D)))) THEN ('CG') WHEN (((_col2 = 'FRESH') and (_col0 <= (333333.0D / UDFToDouble(count_window_1))))) THEN ('TG') WHEN (((_col2 = 'REPEAT') and (_col0 <= (500.0D / UDFToDouble(count_window_1))) and substring(_col1, -2) BETWEEN '60' AND '69')) THEN ('CG') WHEN (((_col2 = 'REPEAT') and (_col0 <= (500.0D / UDFToDouble(count_window_1))) and substring(_col1, -2) BETWEEN '00' AND '45')) THEN ('TG') WHEN (((_col2 = 'REPEAT') and (_col0 <= (500.0D / UDFToDouble(count_window_1))))) THEN ('TG-TELE') ELSE (null) END (type: string) - outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - File Output Operator - compressed: false - Statistics: Num rows: 1 Data size: 920 Basic stats: COMPLETE Column stats: NONE - table: - input format: org.apache.hadoop.mapred.SequenceFileInputFormat - output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat - serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe - - Stage: Stage-0 - Fetch Operator - limit: -1 - Processor Tree: - ListSink - +1