fix(stats): withhold sketch entropy, report its provable ceiling (#108)

docs/src/streaming.md

@@ -162,7 +162,13 @@ Below `exact_threshold` distinct values per path (default 10,000), values are tr
 Past it, the path switches to sketches and its statistics carry `exact: false`:
 
 - `cardinality` comes from **HyperLogLog**: 2 KB per tracked path, a standard error of 2.3% at any magnitude. On a 120,000-distinct field it estimates ~116,000, against the 100 it reported before the sketch existed. It is an estimate either side of the truth, not a bound.
-- `entropy`, `normalized_entropy` and `max_rarity` are rough approximations over the tracked top-k, with no proven bound, and can be off by a factor of two or more — a 120,000-distinct field reports `log2(100)` where the truth is 16.87. Space-Saving admits an evicted item at `min_count + 1`, so its counters over-attribute rather than cleanly grouping the tail, which is why neither a bound nor a tail-mass estimate is recoverable from them. Tracked in [#108](https://github.com/copyleftdev/vajra/issues/108).
+- `entropy` and `normalized_entropy` are **withheld**. Computed over the tracked top-k they gave `log2(k)` — 6.64 against a true 16.87, a 61% error invisible in the number, and not comparable with an exact figure from another path, which is most of what entropy is used for.
+
+  `entropy_upper_bound` is reported in their place: `log2(cardinality)` is a provable ceiling, and with the HyperLogLog estimate behind it a tight one — 16.83 against 16.87 on that field. Text output shows it as `<=16.8275`.
+
+  Three point estimators were measured before choosing a bound. A reservoir sample with the Miller–Madow correction still lands 17% low at n=10,000 and costs ~195 KB per path, a hundred times the HyperLogLog budget. Chao–Shen degenerates exactly where the problem is worst: its coverage term is `1 - f1/n`, which goes to zero when every value is distinct. Neither is worth its cost against a bound that is free and off by 0.3%.
+
+- `max_rarity` remains a figure, because its error direction *is* provable: the rarest tracked value is at least as common as the rarest overall, so its self-information is at most the true maximum. It is a lower bound.
 
 The sketch costs 2 KB per path and is allocated only when a path crosses the threshold — below it, values are counted by identity and the exact count is already known.
 

vajra-cli/src/main.rs

@@ -1796,8 +1796,15 @@ struct StatsOutput {
 #[derive(Serialize)]
 struct StatsPathView {
     path: String,
-    entropy: f64,
-    normalized_entropy: f64,
+    /// Absent when only sketch statistics were available; see
+    /// `entropy_upper_bound`.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    entropy: Option<f64>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    normalized_entropy: Option<f64>,
+    /// Provable ceiling, reported in entropy's place when it is withheld.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    entropy_upper_bound: Option<f64>,
     cardinality: u64,
     total_count: u64,
     max_rarity: f64,
@@ -1858,6 +1865,7 @@ fn build_stats_output(result: &StatsResult) -> StatsOutput {
             exact: stats.exact,
             entropy: stats.entropy,
             normalized_entropy: stats.normalized_entropy,
+            entropy_upper_bound: stats.entropy_upper_bound,
             cardinality: stats.cardinality,
             total_count: stats.total_count,
             max_rarity: stats.max_rarity,
@@ -1928,8 +1936,17 @@ fn cmd_stats(
             for sp in &output.paths {
                 let mut row = vec![
                     sp.path.clone(),
-                    format!("{:.4}", sp.entropy),
-                    format!("{:.4}", sp.normalized_entropy),
+                    // "<=x" rather than a blank: the ceiling is what is known,
+                    // and an empty cell would read as missing data.
+                    sp.entropy.map_or_else(
+                        || {
+                            sp.entropy_upper_bound
+                                .map_or_else(|| "--".to_owned(), |u| format!("<={u:.4}"))
+                        },
+                        |h| format!("{h:.4}"),
+                    ),
+                    sp.normalized_entropy
+                        .map_or_else(|| "--".to_owned(), |n| format!("{n:.4}")),
                     sp.cardinality.to_string(),
                     sp.total_count.to_string(),
                     format!("{:.4}", sp.max_rarity),
@@ -1942,8 +1959,10 @@ fn cmd_stats(
             report.table(summary);
             if any_inexact {
                 report.note(
-                    "approx: past the exact-tracking threshold, so cardinality is a lower \
-                     bound and entropy is a sketch estimate, not a measurement",
+                    "approx: past the exact-tracking threshold. Cardinality is a HyperLogLog \
+                     estimate (~2.3% error); entropy is withheld and its provable ceiling \
+                     log2(cardinality) shown as <=x, because the tracked top-k gives a figure \
+                     off by 61% that cannot be compared with an exact one",
                 );
             }
 

vajra-cli/tests/streaming_bounded.rs

@@ -245,3 +245,74 @@ fn a_non_streamable_input_says_it_was_loaded_whole() {
         String::from_utf8_lossy(&out.stderr)
     );
 }
+
+/// Entropy past the threshold was `log2(top_k)` — 6.64 against a true 16.87,
+/// an error of 61% invisible in the number, and not comparable with an exact
+/// figure from another path. It is withheld, and the provable ceiling
+/// `log2(cardinality)` reported in its place. See #108.
+#[test]
+fn entropy_is_withheld_past_the_threshold_and_bounded_instead() {
+    let dir = tempfile::tempdir().expect("tempdir");
+    // Every value distinct, so true entropy is log2(20_000) = 14.29 and the
+    // old top-k figure would have been log2(100) = 6.64.
+    let path = corpus(dir.path(), "wide.json", 20_000, 20_000, false);
+
+    let streamed = paths_of(&json(&[
+        "stats",
+        &path,
+        "--streaming",
+        "--format",
+        "json",
+        "--quiet",
+    ]));
+    let v = &streamed["$[*].v"];
+
+    assert!(
+        v["entropy"].is_null(),
+        "a figure off by 61% must not be reported as entropy: {v}"
+    );
+    assert!(
+        v["normalized_entropy"].is_null(),
+        "normalized entropy is derived from it and must go too: {v}"
+    );
+
+    let bound = v["entropy_upper_bound"]
+        .as_f64()
+        .expect("the ceiling must be reported in entropy's place");
+
+    let dom = paths_of(&json(&["stats", &path, "--format", "json", "--quiet"]));
+    let truth = dom["$[*].v"]["entropy"].as_f64().expect("true entropy");
+
+    assert!(
+        bound >= truth - 0.5,
+        "the ceiling must not sit below the truth: {bound} < {truth}"
+    );
+    assert!(
+        (bound - truth).abs() < 1.0,
+        "the ceiling should be tight for a near-uniform field: {bound} against {truth}"
+    );
+    // The point being that this is far better than what it replaced.
+    assert!(
+        bound > 10.0,
+        "log2(top_k) was 6.64; the bound must not resemble it: {bound}"
+    );
+}
+
+/// The exact path is untouched: entropy present, no ceiling alongside it.
+#[test]
+fn an_exact_path_reports_entropy_and_no_ceiling() {
+    let dir = tempfile::tempdir().expect("tempdir");
+    let path = corpus(dir.path(), "narrow.json", 4000, 50, false);
+
+    for args in [
+        vec!["stats", &path, "--format", "json", "--quiet"],
+        vec!["stats", &path, "--streaming", "--format", "json", "--quiet"],
+    ] {
+        let v = &paths_of(&json(&args))["$[*].v"];
+        assert!(v["entropy"].as_f64().is_some(), "entropy must be present");
+        assert!(
+            v["entropy_upper_bound"].is_null(),
+            "an exact figure needs no ceiling: {v}"
+        );
+    }
+}

vajra-core/tests/chaos.rs

@@ -197,8 +197,8 @@ fn chaos_huge_array_identical_elements() -> Result<(), Box<dyn std::error::Error
             "expected cardinality 1 for identical values"
         );
         assert!(
-            ps.entropy.abs() < 1e-10,
-            "expected entropy ~0 for identical values, got {}",
+            ps.entropy.is_some_and(|h| h.abs() < 1e-10),
+            "expected entropy ~0 for identical values, got {:?}",
             ps.entropy
         );
     }
@@ -228,8 +228,9 @@ fn chaos_huge_array_unique_elements() -> Result<(), Box<dyn std::error::Error>>
     if let Some(ps) = stats.paths.get(&path) {
         assert_eq!(ps.cardinality, 10_000, "expected cardinality 10000");
         assert!(
-            ps.entropy > 0.0,
-            "expected positive entropy for unique values"
+            ps.entropy.is_some_and(|h| h > 0.0),
+            "expected positive entropy for unique values, got {:?}",
+            ps.entropy
         );
     }
     Ok(())

vajra-core/tests/determinism.rs

@@ -113,7 +113,10 @@ fn determinism_stats_10_runs() -> Result<(), Box<dyn std::error::Error>> {
             .map(|(path, ps)| {
                 format!(
                     "{}:entropy={:.15},norm_entropy={:.15},card={},count={},max_rarity={:.15},top={:?}",
-                    path, ps.entropy, ps.normalized_entropy, ps.cardinality,
+                    path,
+                    ps.entropy.unwrap_or(f64::NAN),
+                    ps.normalized_entropy.unwrap_or(f64::NAN),
+                    ps.cardinality,
                     ps.total_count, ps.max_rarity, ps.top_values
                 )
             })

vajra-essence/src/builder.rs

@@ -162,17 +162,19 @@ fn observations_from_stats(stats: &StatsResult, candidates: &mut Vec<CandidateOb
     for (path, path_stats) in &stats.paths {
         let path_str = path.to_string();
 
-        // High entropy => "informative field"
-        if path_stats.normalized_entropy > 0.8 {
+        // High entropy => "informative field". Skipped when entropy is
+        // unavailable: a claim about a field's information content needs a
+        // figure behind it, and sketch statistics do not supply one (#108).
+        let normalized_entropy = path_stats.normalized_entropy.unwrap_or(0.0);
+        if path_stats.normalized_entropy.is_some_and(|h| h > 0.8) {
             candidates.push(CandidateObservation {
                 path: path_str.clone(),
                 description: format!(
-                    "informative field (normalized entropy {:.3})",
-                    path_stats.normalized_entropy
+                    "informative field (normalized entropy {normalized_entropy:.3})"
                 ),
                 rarity: 0.0,
                 instability: 0.0,
-                entropy_signal: path_stats.normalized_entropy,
+                entropy_signal: normalized_entropy,
                 structural_coverage: 0.0,
                 anomaly_strength: 0.0,
                 concern_relevance: 0.3,

vajra-query/src/error.rs

@@ -30,6 +30,17 @@ pub enum QueryError {
     #[error("stats not available (run with stats context)")]
     StatsRequired,
 
+    /// The statistic exists but could not be computed for this path.
+    #[error("{metric} is unavailable for {path}: {reason}")]
+    MetricUnavailable {
+        /// The metric that was requested.
+        metric: String,
+        /// The path it was requested for.
+        path: String,
+        /// Why it could not be produced.
+        reason: String,
+    },
+
     /// A type mismatch during evaluation.
     #[error("type error: {message}")]
     TypeError {

vajra-query/src/eval.rs

@@ -158,9 +158,9 @@ fn extract_values(
 /// Evaluate a function call.
 fn eval_function(fc: &FunctionCall, ctx: &QueryContext<'_>) -> Result<QueryResult, QueryError> {
     match fc.name.as_str() {
-        "entropy" => eval_stats_fn(fc, ctx, |ps| ps.entropy),
-        "cardinality" => eval_stats_fn(fc, ctx, |ps| ps.cardinality as f64),
-        "rarity" => eval_stats_fn(fc, ctx, |ps| ps.max_rarity),
+        "entropy" => eval_stats_fn(fc, ctx, "entropy", |ps| ps.entropy),
+        "cardinality" => eval_stats_fn(fc, ctx, "cardinality", |ps| Some(ps.cardinality as f64)),
+        "rarity" => eval_stats_fn(fc, ctx, "rarity", |ps| Some(ps.max_rarity)),
         "null_rate" => eval_trie_fn(fc, ctx, |meta| meta.null_rate()),
         "instability" => eval_trie_fn(fc, ctx, |meta| meta.type_instability()),
         "count" => eval_trie_fn(fc, ctx, |meta| meta.count as f64),
@@ -175,7 +175,8 @@ fn eval_function(fc: &FunctionCall, ctx: &QueryContext<'_>) -> Result<QueryResul
 fn eval_stats_fn(
     fc: &FunctionCall,
     ctx: &QueryContext<'_>,
-    extractor: fn(&vajra_stats::PathStats) -> f64,
+    metric: &str,
+    extractor: fn(&vajra_stats::PathStats) -> Option<f64>,
 ) -> Result<QueryResult, QueryError> {
     let path = require_path_arg(fc)?;
     let stats = ctx.stats.ok_or(QueryError::StatsRequired)?;
@@ -189,7 +190,15 @@ fn eval_stats_fn(
             path: path.as_str(),
         })?;
 
-    Ok(QueryResult::Scalar(extractor(path_stats)))
+    // A missing statistic is reported, not substituted. Entropy is withheld
+    // when only a sketch was available, and answering a query with a stand-in
+    // number would be worse than answering with nothing. See #108.
+    let value = extractor(path_stats).ok_or_else(|| QueryError::MetricUnavailable {
+        metric: metric.to_owned(),
+        path: path.as_str(),
+        reason: "only sketch statistics were available for this path".to_owned(),
+    })?;
+    Ok(QueryResult::Scalar(value))
 }
 
 /// Evaluate a trie-based function (null_rate, instability, count).

vajra-stats/src/analyzer.rs

@@ -21,10 +21,28 @@ use crate::renyi::{self, RenyiSpectrum};
 /// Per-path statistics produced by the analyzer.
 #[derive(Debug, Clone)]
 pub struct PathStats {
-    /// Shannon entropy of value distribution.
-    pub entropy: f64,
+    /// Shannon entropy of the value distribution, in bits.
+    ///
+    /// `None` when it cannot be computed honestly. Past the streaming
+    /// exact-tracking threshold only the top-k values are tracked, giving
+    /// `log2(k)` — 6.64 against a true 16.87 on a 120,000-distinct field, an
+    /// error of 61% that is invisible in the number itself. Worse, it is not
+    /// comparable with an exact figure from another path, which is what
+    /// entropy is mostly used for. `entropy_upper_bound` is reported instead.
+    /// See #108.
+    pub entropy: Option<f64>,
     /// Normalized entropy (0 = constant, 1 = uniform).
-    pub normalized_entropy: f64,
+    ///
+    /// `None` whenever `entropy` is, since it is derived from it.
+    pub normalized_entropy: Option<f64>,
+    /// Provable ceiling on `entropy`: a distribution over `n` distinct values
+    /// has at most `log2(n)` bits.
+    ///
+    /// Reported only when `entropy` is `None`, since it adds nothing to an
+    /// exact figure. With the HyperLogLog cardinality estimate behind it this
+    /// is tight where the old point estimate was worst: 16.83 against a true
+    /// 16.87 on the field above, an error of 0.3%.
+    pub entropy_upper_bound: Option<f64>,
     /// Number of distinct values observed.
     ///
     /// A lower bound when `exact` is false.
@@ -42,6 +60,11 @@ pub struct PathStats {
     /// Total observations at this path.
     pub total_count: u64,
     /// Self-information of the rarest value: `-log2(min_p)`.
+    ///
+    /// A lower bound when `exact` is false: the rarest *tracked* value is at
+    /// least as common as the rarest overall, so its self-information is at
+    /// most the true maximum. Unlike entropy that direction is provable, so the
+    /// figure is reported rather than withheld.
     pub max_rarity: f64,
     /// Numeric distribution statistics, if the path contains numeric values.
     pub numeric_stats: Option<NumericStats>,
@@ -126,8 +149,10 @@ impl Analyzer for StatsAnalyzer {
                 PathStats {
                     // The DOM path counts every value by identity.
                     exact: true,
-                    entropy: h,
-                    normalized_entropy: nh,
+                    entropy: Some(h),
+                    normalized_entropy: Some(nh),
+                    // An exact figure needs no ceiling.
+                    entropy_upper_bound: None,
                     cardinality,
                     total_count,
                     max_rarity,
@@ -149,8 +174,8 @@ impl FeatureExtractor for StatsAnalyzer {
 
         for (path, stats) in &result.paths {
             let pf: &mut PathFeatures = features.get_or_create(path);
-            pf.entropy = Some(stats.entropy);
-            pf.normalized_entropy = Some(stats.normalized_entropy);
+            pf.entropy = stats.entropy;
+            pf.normalized_entropy = stats.normalized_entropy;
             pf.cardinality = Some(stats.cardinality);
             pf.count = Some(stats.total_count);
             pf.max_rarity = Some(stats.max_rarity);
@@ -243,7 +268,7 @@ mod tests {
         let a_path = WildcardPath::root().push_key("a");
         let a_stats = result.paths.get(&a_path).ok_or("expected stats for $.a")?;
 
-        assert!((a_stats.entropy - 0.0).abs() < EPS);
+        assert!((a_stats.entropy.unwrap_or(f64::NAN) - 0.0).abs() < EPS);
         assert_eq!(a_stats.cardinality, 1);
         assert_eq!(a_stats.total_count, 1);
         Ok(())
@@ -264,7 +289,7 @@ mod tests {
         // Entropy for p=[3/5, 2/5]
         let expected_h =
             -(3.0 / 5.0 * (3.0_f64 / 5.0).log2()) - (2.0 / 5.0 * (2.0_f64 / 5.0).log2());
-        assert!((stats.entropy - expected_h).abs() < 1e-8);
+        assert!((stats.entropy.unwrap_or(f64::NAN) - expected_h).abs() < 1e-8);
         Ok(())
     }
 
@@ -393,8 +418,8 @@ mod tests {
         let path = WildcardPath::root().push_array_wildcard();
         let stats = result.paths.get(&path).ok_or("missing")?;
 
-        assert!((stats.entropy - 0.0).abs() < EPS);
-        assert!((stats.normalized_entropy - 0.0).abs() < EPS);
+        assert!((stats.entropy.unwrap_or(f64::NAN) - 0.0).abs() < EPS);
+        assert!((stats.normalized_entropy.unwrap_or(f64::NAN) - 0.0).abs() < EPS);
         Ok(())
     }
 
@@ -408,8 +433,8 @@ mod tests {
         let stats = result.paths.get(&path).ok_or("missing")?;
 
         // H = log2(4) = 2.0, normalized = 1.0
-        assert!((stats.entropy - 2.0).abs() < EPS);
-        assert!((stats.normalized_entropy - 1.0).abs() < EPS);
+        assert!((stats.entropy.unwrap_or(f64::NAN) - 2.0).abs() < EPS);
+        assert!((stats.normalized_entropy.unwrap_or(f64::NAN) - 1.0).abs() < EPS);
         Ok(())
     }
 }