fix(vchordrq): estimate filtered candidate selectivity

src/index/vchordrq/am/mod.rs

@@ -292,20 +292,31 @@ pub unsafe extern "C-unwind" fn amcostestimate(
             *index_pages = 1.0;
             return;
         }
-        let selectivity = {
-            use pgrx::pg_sys::{
-                JoinType, add_predicate_to_index_quals, clauselist_selectivity,
-                get_quals_from_indexclauses,
+        // Vchordrq indexes only the vector column, so ordinary filters on
+        // other columns are heap-side quals rather than index quals. We use
+        // the planner's filtered row estimate to decide how many distance
+        // candidates we likely need to produce enough survivors for LIMIT.
+        //
+        // Keep this separate from the value returned as `indexSelectivity`:
+        // `cost_index()` interprets that output as the fraction of parent
+        // table rows the index scan retrieves, not the fraction surviving all
+        // filters. With LIMIT, the retrieved fraction is better represented
+        // by the candidate count computed below.
+        let (total_rows, filter_selectivity) = {
+            let baserel = (*index_opt_info).rel;
+            let total_rows = (*baserel).tuples;
+            let param_info = (*path).path.param_info;
+            let filtered_rows = if !param_info.is_null() {
+                (*param_info).ppi_rows
+            } else {
+                (*baserel).rows
             };
-            let index_quals = get_quals_from_indexclauses((*path).indexclauses);
-            let selectivity_quals = add_predicate_to_index_quals(index_opt_info, index_quals);
-            clauselist_selectivity(
-                root,
-                selectivity_quals,
-                (*(*index_opt_info).rel).relid as _,
-                JoinType::JOIN_INNER,
-                std::ptr::null_mut(),
-            )
+            let filter_selectivity = if total_rows > 0.0 {
+                (filtered_rows / total_rows).clamp(1e-9, 1.0)
+            } else {
+                1.0
+            };
+            (total_rows, filter_selectivity)
         };
         // index exists
         if !(*index_opt_info).hypothetical {
@@ -367,18 +378,31 @@ pub unsafe extern "C-unwind" fn amcostestimate(
                 pages += cost.cells[0] as f64;
                 pages
             };
-            let next_count =
-                f64::max(1.0, (*root).limit_tuples) * f64::min(1000.0, 1.0 / selectivity);
+            // `next_count` represents candidates we expect to process to
+            // surface `limit_tuples` survivors after filter rejection. Clamp
+            // by `node_count` so the estimate cannot exceed the candidates
+            // the IVF visits at the configured probe count.
+            let next_count = if (*root).limit_tuples > 0.0 {
+                ((*root).limit_tuples * f64::min(1000.0, 1.0 / filter_selectivity))
+                    .min(node_count)
+            } else {
+                node_count
+            };
+            let scan_selectivity = if total_rows > 0.0 {
+                (next_count / total_rows).clamp(1e-9, 1.0)
+            } else {
+                1.0
+            };
             *index_startup_cost = 0.001 * node_count;
             *index_total_cost = 0.001 * node_count + next_count;
-            *index_selectivity = selectivity;
+            *index_selectivity = scan_selectivity;
             *index_correlation = 0.0;
             *index_pages = page_count;
             return;
         }
         *index_startup_cost = 0.0;
         *index_total_cost = 0.0;
-        *index_selectivity = selectivity;
+        *index_selectivity = filter_selectivity;
         *index_correlation = 0.0;
         *index_pages = 1.0;
     }