HIVE-29729: AIBOE in HiveFilterProjectTransposeRule during CBO planni…

ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveFilterProjectTransposeRule.java

@@ -338,11 +338,14 @@ public void visit(RelNode node, int ordinal, RelNode parent) {
         if (node instanceof Filter) {
           check((Filter) node);
         } else if (node instanceof Project) {
+          Project project = (Project) node;
           RexNode condition = HiveRelOptUtil.pushPastProjectUnlessBloat(
-              filterCondition, (Project) node, bloat);
-          if (condition != null) {
-            filterCondition = condition;
+              filterCondition, project, bloat);
+          if (condition == null) {
+            // the condition could not be pushed, so bail out
+            return;
           }
+          filterCondition = condition;
         } else {
           // we do not support other operators for now
           return;

ql/src/test/queries/clientpositive/cbo_filter_pushdown_windowing_notnull.q

@@ -0,0 +1,16 @@
+-- HIVE-29729: an IS NOT NULL predicate above nested windowing (OVER) Projects must
+-- not trip the RedundancyChecker in HiveFilterProjectTransposeRule.
+set hive.cbo.fallback.strategy=NEVER;
+
+create table tab1 (id string);
+
+select rnk2
+from (
+  select
+    row_number() over(order by rnk) as rnk2
+  from (
+    select count(*) over() as rnk
+    from (select count(*) from tab1) t
+  ) t2
+) t3
+where rnk2 is not null;

ql/src/test/results/clientpositive/llap/cbo_filter_pushdown_windowing_notnull.q.out

@@ -0,0 +1,35 @@
+PREHOOK: query: create table tab1 (id string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@tab1
+POSTHOOK: query: create table tab1 (id string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@tab1
+PREHOOK: query: select rnk2
+from (
+  select
+    row_number() over(order by rnk) as rnk2
+  from (
+    select count(*) over() as rnk
+    from (select count(*) from tab1) t
+  ) t2
+) t3
+where rnk2 is not null
+PREHOOK: type: QUERY
+PREHOOK: Input: default@tab1
+#### A masked pattern was here ####
+POSTHOOK: query: select rnk2
+from (
+  select
+    row_number() over(order by rnk) as rnk2
+  from (
+    select count(*) over() as rnk
+    from (select count(*) from tab1) t
+  ) t2
+) t3
+where rnk2 is not null
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@tab1
+#### A masked pattern was here ####
+1