From 5d7dcdd9e8a39fc6258e5375e687abe7d6a6483d Mon Sep 17 00:00:00 2001 From: igerber Date: Sun, 19 Jul 2026 16:44:27 -0400 Subject: [PATCH] feat: MMM calibration export (diff_diff.mmm) - PyMC-Marketing lift tests + Meridian ROI priors Interop builders that assemble Marketing Mix Model calibration inputs from experiment results. Explicit-in / validated-out: the caller supplies the already-scoped incremental outcome and its SE (aggregated to the population and window one MMM row represents), and diff-diff assembles the target schema, enforces each consumer's guards, converts to the lognormal parameterization, and pools. It does NOT rescale a headline ATT - that reconciliation needs the MMM's row granularity, time window, and outcome scale, which the exporter cannot see. Pure numpy/pandas; imports no MMM package; introspects no result object, so the module is purely additive (touches no estimator). - to_pymc_marketing_lift_test(channel, x, delta_x, delta_y, sigma, dims=, on_wrong_sign=): builds the lift-test DataFrame consumed by pymc-marketing's MMM.add_lift_test_measurements (prophetverse-compatible). Guards: sigma>0, delta_x!=0, x>=0, x+delta_x>=0, finite delta_y; wrong-sign (NonMonotonicError) AND zero-lift (degenerate for the Gamma lift likelihood) share one on_wrong_sign policy; dims reserved-column + shared-keys validation. - to_meridian_roi_prior(incremental_outcome, incremental_outcome_se, spend, parameter="roi_m"|"mroi_m", se_widening=): builds Google Meridian lognormal priors (MeridianROIPrior); mu/sigma match meridian's lognormal_dist_from_mean_std (verified 1.7.0), spend-weighted multi-experiment pooling, non-positive-ROI and finiteness guards, and a channel- and time-scoped .to_code() snippet that sets media_prior_type. roi_m vs mroi_m selects the estimand and each parameter's Meridian default for non-experiment channels. - 43 behavioral tests (schema, all guards, sign/zero policy, lognormal roundtrip via scipy, pooling, to_code scoping, realistic DiD/CS workflow) + end-to-end smoke verified against real pymc-marketing 0.19.4 (frame accepted into the model graph; wrong-signed row raises NonMonotonicError upstream). - Docs: REGISTRY interop section, api/mmm.rst, llms.txt + llms-full.txt, README line, references.rst, doc-deps.yaml, CHANGELOG; TODO/DEFERRED follow-up rows (Meridian mask builder, PR-B tutorial, post-4.0 result-derived scaling via results.aggregate()). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014wAAYSVo1Yf3XMcxNMrqsj --- CHANGELOG.md | 20 + DEFERRED.md | 1 + README.md | 1 + TODO.md | 2 + diff_diff/__init__.py | 9 + diff_diff/guides/llms-full.txt | 83 ++++- diff_diff/guides/llms.txt | 1 + diff_diff/mmm.py | 653 +++++++++++++++++++++++++++++++++ docs/api/index.rst | 15 + docs/api/mmm.rst | 124 +++++++ docs/doc-deps.yaml | 21 ++ docs/methodology/REGISTRY.md | 27 ++ docs/references.rst | 15 + tests/test_mmm.py | 471 ++++++++++++++++++++++++ 14 files changed, 1442 insertions(+), 1 deletion(-) create mode 100644 diff_diff/mmm.py create mode 100644 docs/api/mmm.rst create mode 100644 tests/test_mmm.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a13b6d16..696eb2df2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -78,6 +78,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 standard rename rows (M-088/M-089) under the spec's missed-rename clause. Docs + ledger + test-count updates only - **no public API or numerical behavior change.** +- **MMM calibration export (`diff_diff.mmm`)** - interop builders that assemble + Marketing Mix Model calibration inputs from experiment results, with no MMM + package dependency and no result introspection (purely additive). Design is + explicit-in / validated-out: the caller supplies the already-scoped + incremental outcome and its SE (aggregated to the population and window one + MMM row represents), and diff-diff assembles the schema, enforces each + consumer's guards, and converts to the target parameterization - it does not + rescale a headline ATT, because that reconciliation needs the MMM's row + structure and outcome scale it cannot see. + `to_pymc_marketing_lift_test(channel, x, delta_x, delta_y, sigma, dims=, + on_wrong_sign=)` builds the PyMC-Marketing/prophetverse lift-test DataFrame + with sign/zero (Gamma-likelihood)/positivity guards. + `to_meridian_roi_prior(incremental_outcome, incremental_outcome_se, spend, + parameter="roi_m"|"mroi_m", se_widening=)` builds Google Meridian lognormal + ROI/mROI priors (`MeridianROIPrior` with `mu`/`sigma` matching Meridian's + `lognormal_dist_from_mean_std`, spend-weighted multi-experiment pooling, and a + channel- and time-scoped `.to_code()` snippet that sets `media_prior_type`). + Deriving totals directly from a fitted result is deferred to the post-4.0 + `results.aggregate()` layer. + - **Reviewer-eval harness: N-arm matrix, blinded grading, corpus grown 2 -> 11 (`tools/reviewer-eval/`, prep for the GPT-5.6 reviewer evaluation).** `config/configs.json` moves from the two-arm `control`/`candidate` shape to an diff --git a/DEFERRED.md b/DEFERRED.md index 2bba30ce8..5b74f6ee9 100644 --- a/DEFERRED.md +++ b/DEFERRED.md @@ -70,6 +70,7 @@ For survey-specific limitations (`NotImplementedError` paths), see the | Issue | Location | PR | Priority | |-------|----------|----|----------| +| MMM interop: result-derived scaling for the exporters - let a fitted result produce the scoped incremental outcome + SE directly (via the post-4.0 `results.aggregate()` layer, where the estimator owns its aggregation weights/balance/survey masses), so callers need not hand-scale the ATT. v1 is explicit-in by design; this is the seamless follow-up. | `diff_diff/mmm.py`, `docs/v4-design.md` | mmm-interop | Low | | `SyntheticControl` fit-snapshot residency (`_SyntheticControlFitSnapshot`) — **investigated 2026-07-07, parked**: the snapshot ALIASES the fit's own working pivots (zero extra construction cost); the retained residency implements the documented freeze contract (post-fit mutation of estimator inputs must not change `in_space_placebo()` / `leave_one_out()` / conformal output on an already-returned results object, and `__getstate__` already excludes it from pickles). A compact array representation saves only pandas overhead (the float panel dominates); releasing residency needs new API surface (`release`/opt-out flag) or a freeze-contract change. Revisit on user demand for very large donor panels. | `synthetic_control.py`, `synthetic_control_results.py` | follow-up | Low | | Stratified survey-PSU multiplier-weight draw-tiling — **investigated 2026-07-07, parked**: the stratified generator (`generate_survey_multiplier_weights_batch`) consumes ONE sequential rng stream stratum-major (`rng.choice(size=(n_bootstrap, n_h))` per stratum, then lonely-PSU pooling), so draw-chunked assembly CANNOT reproduce the stream bit-identically (contra the old row's parenthetical) — it would need per-stratum generator state skipping (PCG64.advance + per-weight-type variate accounting; fragile) or a stream-layout change (MC-level SE changes → baseline/golden recapture + REGISTRY note). Stratified designs have few PSUs, so the full `(n_bootstrap × n_psu)` matrix rarely matters; unstratified (the large-`n_units` case) is already tiled. Revisit only if a large-PSU stratified design hits memory, as a documented stream change. | `diff_diff/bootstrap_chunking.py::iter_survey_multiplier_weight_blocks` | follow-up | Low | | ChangesInChanges FULL Melly-Santangelo covariate estimator (monotonized integrated-indicator conditional CDFs, treated-post `F_{X|11}` integration, exchangeable bootstrap with variance-weighted KS bands, tail trimming, pre-period specification test). The qte-`xformla` simplified form of the MS pipeline SHIPPED in the covariates PR (`covariates=` on both estimators, parity-tested vs qte 1.3.1); this row's earlier "No R parity target exists" claim was WRONG and is corrected in the MS review doc. The full estimator has no reference implementation (the MS Stata code is the only one; distinct from Kranker's `cic`) and would need simulation-based validation. Reviewed: `docs/methodology/papers/melly-santangelo-2015-review.md`. | `diff_diff/changes_in_changes.py` | #682 | Low | diff --git a/README.md b/README.md index 54b4ab68e..be7532e32 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,7 @@ Full guide: `diff_diff.get_llm_guide("practitioner")`. - [Honest DiD](https://diff-diff.readthedocs.io/en/stable/api/honest_did.html) - Rambachan & Roth (2023) sensitivity analysis: robust CI under PT violations, breakdown values - [Pre-Trends Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/pretrends.html) - Roth (2022) minimum detectable violation and power curves - [Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/power.html) - analytical and simulation-based MDE, sample size, power curves for study design +- [MMM Calibration Export](https://diff-diff.readthedocs.io/en/stable/api/mmm.html) - convert experiment results into MMM calibration inputs: PyMC-Marketing lift-test frames and Google Meridian lognormal ROI priors - Conley spatial HAC SE (`vcov_type="conley"`) on cross-sectional `LinearRegression` / `compute_robust_vcov` plus panel `DifferenceInDifferences` / `MultiPeriodDiD` / `TwoWayFixedEffects` (with `conley_lag_cutoff` for within-unit Bartlett temporal HAC) - Conley (1999) spatial-correlation-aware SEs with parity vs R `conleyreg` on cross-sectional + panel fixtures, optional combined spatial + cluster product kernel via explicit `cluster=`, auto-activating sparse k-d-tree fast path for `n > 5_000` ## Survey Support diff --git a/TODO.md b/TODO.md index db052c8da..791ab1055 100644 --- a/TODO.md +++ b/TODO.md @@ -43,5 +43,7 @@ generic sparse-FE, QR+SVD rank-detection redundancy, `check_finite` bypass — m |-------|----------|--------|--------|----------| | Align the four legacy dataset loaders (`load_card_krueger`, `load_castle_doctrine`, `load_divorce_laws`, `load_mpdta`) with the loud-fallback pattern of `load_prop99`/`load_walmart`: `UserWarning` + `df.attrs["source"]` marker on synthetic fallback (currently silent), plus optional checksum pinning for the CSV downloads. **Upgraded to a live defect 2026-07-13: the `causaldata/causal_datasets` GitHub repo backing castle/card_krueger/divorce is dead (404), so those loaders silently serve synthetic data everywhere - needs loud fallback + replacement sources.** | `diff_diff/datasets.py` | LWDiD precursor | Quick | Medium | | Tighten the mypy suppressions that back the enforced-zero posture: burn down `prep_dgp`'s per-module `[index]` override (needs a None-vs-array restructure that preserves the seeded RNG stream), and evaluate re-enabling the globally disabled codes (`arg-type`, `return-value`, `var-annotated`, `assignment`) one at a time — `assignment` alone hid several real annotation drifts found during the 2026-07 triage. | `pyproject.toml` `[tool.mypy]`, `diff_diff/prep_dgp.py` | lint-CI | Mid | Low | +| MMM interop follow-up: Meridian `roi_calibration_period` mask builder - accept the MMM's time index + channel order and emit the boolean `(n_media_times, n_media_channels)` mask so `.to_code()` scopes the prior to the experiment window automatically (today the caller passes a mask expression / `full_model_window=True`). | `diff_diff/mmm.py` | mmm-interop | Quick | Low | +| MMM interop PR-B: calibration tutorial notebook (fit DiD/CS -> scope -> `to_pymc_marketing_lift_test` / `to_meridian_roi_prior`) + a `llms-practitioner.txt` Step 8 pointer to the exporters as the MMM hand-off. | `docs/tutorials/`, `diff_diff/guides/llms-practitioner.txt` | mmm-interop | Mid | Low | | Tracking-file contract guard test: reject NEW active deferred-work pointers at `TODO.md` (deferred rows live in `DEFERRED.md`; allowlist for historical/past-tense prose and actionable-row pointers) and assert rows cross-linking a `docs/v4-deprecations.yaml` `M-xxx` id don't restate ledger status. Origin: tracking-split local review R2. | `tests/`, `TODO.md`, `DEFERRED.md` | tracking-split | Quick | Low | | Real-data CI canary for dataset-backed replication tests: `test_methodology_lwdid.py`'s Prop 99 / Walmart goldens skip (visibly) when loaders fall back to synthetic; add a lane or canary asserting `df.attrs["source"] == "lwdid_ssc_ancillary"` in CI so network regressions cannot silently de-gate the replication tests. Pairs with the loader-fallback repair row above. | `tests/test_methodology_lwdid.py`, `.github/workflows/` | LWDiD validation suite | Quick | Low | diff --git a/diff_diff/__init__.py b/diff_diff/__init__.py index 17f10341e..a526901e7 100644 --- a/diff_diff/__init__.py +++ b/diff_diff/__init__.py @@ -154,6 +154,11 @@ ) from diff_diff.lpdid import LPDiD from diff_diff.lpdid_results import LPDiDResults +from diff_diff.mmm import ( + MeridianROIPrior, + to_meridian_roi_prior, + to_pymc_marketing_lift_test, +) from diff_diff.power import ( PowerAnalysis, PowerResults, @@ -566,6 +571,10 @@ "Alert", "OutcomeShape", "TreatmentDoseShape", + # MMM calibration export (interop) + "to_pymc_marketing_lift_test", + "to_meridian_roi_prior", + "MeridianROIPrior", # LLM guide accessor "get_llm_guide", ] diff --git a/diff_diff/guides/llms-full.txt b/diff_diff/guides/llms-full.txt index 74b2de0f9..5c92b3aa6 100644 --- a/diff_diff/guides/llms-full.txt +++ b/diff_diff/guides/llms-full.txt @@ -2042,7 +2042,6 @@ sample_result = compute_sample_size(effect_size=0.5, sigma=1.0) # Simulation-based power sim_result = simulate_power( - n_units=200, n_periods=8, treatment_period=4, effect_sizes=[0.1, 0.5, 1.0, 2.0], n_simulations=500, seed=42, ) @@ -2731,3 +2730,85 @@ cross-period aggregations are enumerated in current release; see that document for phrasing rules, the no-traffic-light decision, unit-translation policy, and schema stability policy. + +## MMM Calibration Export + +Interop builders converting experiment results into Marketing Mix Model +(MMM) calibration inputs. Pure numpy/pandas - no MMM package is imported, +no result object is introspected (the module is purely additive). Design: +EXPLICIT IN, VALIDATED OUT. Reconciling an experiment estimate to a +calibration input needs the target MMM's row granularity (per-geo vs +national), its time window, and the outcome scale (additive levels vs +log/rate/share) - none of which diff-diff can see - so the CALLER supplies +the already-scoped incremental outcome and its SE (read off summary(), +aggregated to the population/window one MMM row represents). diff-diff +assembles the schema, enforces each consumer's guards, converts to the +lognormal parameterization, pools, and emits snippets. Deriving totals +from a fit is deferred to the post-4.0 results.aggregate() layer, where +the estimator owns its aggregation weights/balance/survey masses. + +```python +from diff_diff import ( + SyntheticDiD, + to_pymc_marketing_lift_test, + to_meridian_roi_prior, +) + +# Single-treated-geo experiment: with exactly ONE treated geo the SDID ATT +# is that geo's lift, so a geo-labelled row is sound. For MULTIPLE treated +# geos the pooled ATT is an average - export each geo's own effect or omit +# the geo dim and match an aggregate lift to aggregate spend. +result = SyntheticDiD().fit(panel, outcome="revenue", treatment="treated", + unit="geo", time="week") + +# PyMC-Marketing / prophetverse lift-test frame. Pass the scoped effect. +df_lift = to_pymc_marketing_lift_test( + channel="tv", # must match the MMM's channel_columns + x=50_000.0, # baseline spend for this row's scope + delta_x=20_000.0, # spend change (nonzero; negative go-dark, + # with x + delta_x >= 0) + delta_y=result.att, # measured lift, scoped to this row + sigma=result.se, # its SE (finite, > 0) + dims={"geo": "US-CA"}, # optional model-dim coordinate (one treated geo) + on_wrong_sign="raise", # raise|drop|keep - PyMC rejects + # sign(delta_y)!=sign(delta_x) AND delta_y==0 + # (degenerate for its Gamma lift likelihood) +) + +# Google Meridian lognormal prior. Caller aggregates the ATT to a total +# incremental outcome over the treated population/window and its SE. +prior = to_meridian_roi_prior( + incremental_outcome=180_000.0, # total incremental revenue + incremental_outcome_se=45_000.0, # its SE + spend=200_000.0, # channel spend the outcome is over + parameter="roi_m", # roi_m (full-spend/zero-spend return) + # | mroi_m (marginal return) + se_widening=1.5, # >=1 for transferability skepticism +) +prior.roi_mean, prior.roi_sd # pooled ROI moments +prior.mu, prior.sigma # LogNormal params (match Google's + # lognormal_dist_from_mean_std) +prior.to_dict() # JSON-ready +print(prior.to_code( # ready-to-paste PriorDistribution + + channel="tv", # ModelSpec; roi_m/mroi_m is per-channel + media_channels=["search", "tv"], # so channel scope is required (vector + # prior in model channel order; other + # channels keep the Meridian default), + roi_calibration_period="mask", # AND time scope (mask expr) or + # full_model_window=True; sets +)) # media_prior_type accordingly. +``` + +Key points: +- Lift frame: one row per experiment, columns [channel, *dims, x, delta_x, + delta_y, sigma]; guards on sigma>0, delta_x!=0, x>=0, x+delta_x>=0, + finite delta_y; dims must not collide with reserved columns and must + share one key set. on_wrong_sign governs wrong-sign AND zero-lift rows + (drop-all raises; keep warns the frame is not valid PyMC input). +- Meridian: roi = incremental_outcome/spend, spend-weighted pooling, + roi_sd = sqrt(sum((w_i*sd_i)^2)) (independence assumed - widen via + se_widening); non-positive pooled ROI raises (lognormal positivity); + all scaled/pooled outputs re-validated finite-positive. +- parameter="roi_m" vs "mroi_m" picks the Meridian estimand + its default + for non-experiment channels (LogNormal(0.2,0.9) vs LogNormal(0.0,0.5)) + and the emitted media_prior_type ("roi" vs "mroi"). diff --git a/diff_diff/guides/llms.txt b/diff_diff/guides/llms.txt index efeedc5f5..1cd5c2e8c 100644 --- a/diff_diff/guides/llms.txt +++ b/diff_diff/guides/llms.txt @@ -83,6 +83,7 @@ Full practitioner guide: call `diff_diff.get_llm_guide("practitioner")` - [Honest DiD](https://diff-diff.readthedocs.io/en/stable/api/honest_did.html): Rambachan & Roth (2023) sensitivity analysis — robust CI under parallel trends violations, breakdown values - [Pre-Trends Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/pretrends.html): Roth (2022) Section II.A-B no-individually-significant (NIS) box-probability pretest power + minimum detectable violation; `pretest_form='nis'` (default) implements the paper's primary form, `pretest_form='wald'` retained as paper-supported alternative (Propositions 1+3+4 all apply); linear-violation MDV in Roth's γ units when relative-time labels are threaded through `fit()`; full Σ_22 routing on non-bootstrap CallawaySantAnna and SunAbraham adapters - [Power Analysis](https://diff-diff.readthedocs.io/en/stable/api/power.html): Analytical and simulation-based power analysis — MDE, sample size, power curves for study design +- [MMM Calibration Export](https://diff-diff.readthedocs.io/en/stable/api/mmm.html): Assemble Marketing Mix Model calibration inputs from experiment results (explicit-in / validated-out - the caller passes the already-scoped incremental outcome + SE, the module does NOT rescale a headline ATT). `to_pymc_marketing_lift_test(channel, x, delta_x, delta_y, sigma, dims=, on_wrong_sign=)` builds the PyMC-Marketing/prophetverse lift-test DataFrame with sign/zero/positivity guards. `to_meridian_roi_prior(incremental_outcome, incremental_outcome_se, spend, parameter="roi_m"|"mroi_m", se_widening=)` builds Google Meridian lognormal ROI priors (spend-weighted pooling, lognormal parity with `lognormal_dist_from_mean_std`, channel- and time-scoped `.to_code()` snippet setting `media_prior_type`). Pure numpy/pandas; imports no MMM package; does not introspect result objects. Deriving totals from a fit is deferred to the post-4.0 `results.aggregate()` layer. - Conley spatial HAC SE (`vcov_type="conley"`) on cross-sectional `LinearRegression` / `compute_robust_vcov` PLUS panel `DifferenceInDifferences` / `MultiPeriodDiD` / `TwoWayFixedEffects` (with `conley_lag_cutoff=` for within-unit Bartlett temporal HAC) — Conley (1999) spatial-correlation-aware SEs with haversine/euclidean/callable distance metric and Bartlett/uniform spatial kernel; panel path uses the R `conleyreg`-form block-decomposed sandwich (within-period spatial + within-unit Bartlett serial, same-time excluded); parity vs R `conleyreg` (Düsterhöft 2021) on cross-sectional AND panel `lag_cutoff > 0` fixtures. Combining with explicit `cluster=` applies the combined spatial + cluster product kernel `K_total[i,j] = K_space · 1{c_i = c_j}` (cluster must be constant within each unit across periods on the panel path; validator-enforced). DiD takes `unit=` as a fit-time kwarg when `vcov_type="conley"` (not on `__init__`). Sparse k-d-tree fast path auto-activates for `n > 5_000` with bartlett kernel + haversine/euclidean metric ## Tutorials diff --git a/diff_diff/mmm.py b/diff_diff/mmm.py new file mode 100644 index 000000000..588d5c863 --- /dev/null +++ b/diff_diff/mmm.py @@ -0,0 +1,653 @@ +"""Assemble Marketing Mix Model (MMM) calibration inputs from experiment results. + +MMM practitioners calibrate their models against experimental evidence. The two +dominant Python MMM frameworks consume that evidence in different shapes: + +- **PyMC-Marketing** (and prophetverse, which mirrors the same schema) ingests a + *lift-test* DataFrame with columns ``channel``, optional model dims (e.g. ``geo``), + ``x`` (baseline channel spend), ``delta_x`` (spend change during the test), + ``delta_y`` (measured incremental outcome), and ``sigma`` (the experiment's standard + error), which it scores against the model's saturation curve via + ``MMM.add_lift_test_measurements``. See + https://www.pymc-marketing.io/en/stable/notebooks/mmm/mmm_lift_test.html. +- **Google Meridian** has no experiment-ingestion API; calibration means setting a + lognormal prior per channel - ``roi_m`` for the return of a channel's full spend + (a full-holdout/zero-spend estimand), ``mroi_m`` for a marginal return. Google's + documented workflow maps the experiment's ROI point estimate to the prior mean and + its standard error to the prior standard deviation, converted to lognormal + ``(mu, sigma)`` via the closed form used by + ``meridian.model.prior_distribution.lognormal_dist_from_mean_std``: + ``mu = ln(m) - ln((s/m)^2 + 1)/2``, ``sigma = sqrt(ln((s/m)^2 + 1))``. See + https://developers.google.com/meridian/docs/advanced-modeling/set-custom-priors-past-experiments. + +**Design: explicit in, validated out.** These functions do NOT rescale a result's +headline ATT. Reconciling an experiment's estimate to a calibration input requires +context diff-diff cannot see - the target MMM's row granularity (per-geo vs national), +its time window, and the outcome's scale (additive levels vs a log/rate/share) - so +the CALLER supplies the already-scoped incremental outcome and its standard error (the +numbers they read off a fitted result's ``summary()``, aggregated to the population and +window their MMM row represents). diff-diff does only what it can verify: assemble the +exact target schema, enforce the sign/positivity/monotonicity guards each consumer +requires, convert to the lognormal parameterization (with parity to Google's closed +form), pool multiple experiments, and emit ready-to-paste snippets. It is pure +numpy/pandas and never imports an MMM package. +""" + +from __future__ import annotations + +import ast +import math +import warnings +from dataclasses import dataclass, field +from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union + +import numpy as np +import pandas as pd + +__all__ = ["MeridianROIPrior", "to_meridian_roi_prior", "to_pymc_marketing_lift_test"] + +_WRONG_SIGN_POLICIES = ("raise", "drop", "keep") +_LIFT_TEST_RESERVED = frozenset({"channel", "x", "delta_x", "delta_y", "sigma"}) + +# Meridian prior parameters this exporter can target, with each one's Meridian +# default LogNormal(mu, sigma) per channel (verified against +# meridian/model/prior_distribution.py at 1.7.0): roi_m is the return on a +# channel's full spend (zero-spend counterfactual), mroi_m the marginal return. +# Channels without an experiment keep the default in the vector snippet. +_MERIDIAN_PARAM_DEFAULTS = {"roi_m": (0.2, 0.9), "mroi_m": (0.0, 0.5)} + +# The .to_code() templates are pinned against google-meridian 1.7.0 (2026-06); they +# are convenience snippets, not a programmatic contract. Meridian's roi_m/mroi_m have +# batch shape n_media_channels; a scalar LogNormal broadcasts to EVERY channel, so the +# scalar template is gated behind an explicit single_channel opt-in. +_MERIDIAN_SINGLE_CHANNEL_TEMPLATE = """\ +# Generated by diff_diff.mmm.to_meridian_roi_prior (template pinned to Meridian 1.7.0) +# TensorFlow-substrate snippet; for JAX-backed Meridian use +# `import tensorflow_probability.substrates.jax as tfp` instead. +# SINGLE-CHANNEL MODEL ONLY: a scalar {param} prior broadcasts to every media channel. +# For a multi-channel model, regenerate with to_code(channel=..., media_channels=[...]). +import tensorflow_probability as tfp +from meridian.model import prior_distribution, spec + +roi_prior = tfp.distributions.LogNormal({mu!r}, {sigma!r}, name="{param}") +prior = prior_distribution.PriorDistribution({param}=roi_prior) +model_spec = spec.ModelSpec( + prior=prior, + media_prior_type="{prior_type}", + # {window_note} + roi_calibration_period={calibration_period}, +) +""" + +_MERIDIAN_MULTI_CHANNEL_TEMPLATE = """\ +# Generated by diff_diff.mmm.to_meridian_roi_prior (template pinned to Meridian 1.7.0) +# TensorFlow-substrate snippet; for JAX-backed Meridian use +# `import tensorflow_probability.substrates.jax as tfp` instead. +# Channel order MUST match your Meridian InputData media channel order exactly: +# {channels} +# {channel!r} carries the experiment-informed prior; the other channels keep +# Meridian's default {param} prior LogNormal({default_mu}, {default_sigma}). +import tensorflow_probability as tfp +from meridian.model import prior_distribution, spec + +mu = {mu_vector!r} +sigma = {sigma_vector!r} +roi_prior = tfp.distributions.LogNormal(mu, sigma, name="{param}") +prior = prior_distribution.PriorDistribution({param}=roi_prior) +model_spec = spec.ModelSpec( + prior=prior, + media_prior_type="{prior_type}", + # {window_note} + roi_calibration_period={calibration_period}, +) +""" + + +def _is_sequence(value: Any) -> bool: + """True for list-like containers (not strings, not mappings).""" + return isinstance(value, (list, tuple, np.ndarray, pd.Series)) + + +def _broadcast(name: str, value: Any, n: int) -> List[Any]: + """Broadcast a scalar to n rows, or validate a sequence's length.""" + if _is_sequence(value): + values = list(value) + if len(values) != n: + raise ValueError( + f"{name} has length {len(values)} but {n} experiment(s) were given; " + f"pass one value per experiment or a single scalar" + ) + return values + return [value] * n + + +def _seq_len(*values: Any) -> int: + """Number of experiments implied by the first sequence argument (else 1).""" + for v in values: + if _is_sequence(v): + n = len(list(v)) + if n == 0: + raise ValueError("empty sequence given; provide at least one experiment") + return n + return 1 + + +def _finite_positive(name: str, value: Any, index: int) -> float: + v = float(value) + if not math.isfinite(v) or v <= 0: + raise ValueError(f"{name} must be finite and > 0; got {value!r} for experiment[{index}]") + return v + + +def _normalize_dims( + dims: Optional[Union[Mapping[str, str], Sequence[Mapping[str, str]]]], n: int +) -> Tuple[List[str], List[Mapping[str, str]]]: + """Broadcast dims mappings and pin a single shared key set / column order.""" + if dims is None: + return [], [] + if isinstance(dims, Mapping): + rows: List[Mapping[str, str]] = [dims] * n + else: + rows = list(_broadcast("dims", dims, n)) + for i, row in enumerate(rows): + if not isinstance(row, Mapping): + raise TypeError( + f"dims must be a mapping or a sequence of mappings (e.g. " + f"{{'geo': 'US-CA'}}); dims[{i}] is {type(row).__name__}" + ) + dim_cols = list(rows[0].keys()) + key_set = set(dim_cols) + reserved = key_set & _LIFT_TEST_RESERVED + if reserved: + raise ValueError( + f"dims keys {sorted(reserved)} collide with reserved lift-test columns " + f"{sorted(_LIFT_TEST_RESERVED)}; rename the model dimension" + ) + for i, row in enumerate(rows): + if set(row.keys()) != key_set: + raise ValueError( + f"dims mappings must share one key set across rows; dims[{i}] has keys " + f"{sorted(row.keys())}, expected {sorted(key_set)}" + ) + return dim_cols, rows + + +def to_pymc_marketing_lift_test( + *, + channel: Union[str, Sequence[str]], + x: Union[float, Sequence[float]], + delta_x: Union[float, Sequence[float]], + delta_y: Union[float, Sequence[float]], + sigma: Union[float, Sequence[float]], + dims: Optional[Union[Mapping[str, str], Sequence[Mapping[str, str]]]] = None, + on_wrong_sign: str = "raise", +) -> pd.DataFrame: + """Assemble a PyMC-Marketing lift-test DataFrame from scoped experiment results. + + Produces one row per experiment with columns ``channel``, any ``dims`` columns, + ``x``, ``delta_x``, ``delta_y``, ``sigma`` - the exact schema consumed by + ``pymc_marketing.mmm.MMM.add_lift_test_measurements`` (prophetverse's lift-test + API accepts the same shape). All values stay in original data units + (PyMC-Marketing rescales internally). + + **The caller supplies the scoped effect.** ``delta_y`` and ``sigma`` are the + measured incremental outcome and its standard error, already aggregated to the + population and time window ONE target-MMM row represents. This function does not + rescale an ATT, because the reconciliation needs the MMM's row granularity and + outcome scale, which it cannot see. Read the effect and SE off your fitted + result's ``summary()`` and scope them yourself: PyMC-Marketing scores one row's + ``delta_y`` against ``saturation(x + delta_x) - saturation(x)``, so ``x``, + ``delta_x``, ``delta_y``, and ``sigma`` must all describe the SAME observation + (same channel, same population, same period span, additive-level outcome). + + Parameters + ---------- + channel : str or sequence of str + MMM channel name(s); must match the target model's ``channel_columns``. + x : float or sequence of float + Baseline channel spend for the row's observation, in original spend units. + delta_x : float or sequence of float + Spend change during the experiment (nonzero; negative for go-dark/holdout + tests, with ``x + delta_x >= 0``), in the same units as ``x``. + delta_y : float or sequence of float + Measured incremental outcome for the SAME observation as ``x``/``delta_x``, + in original outcome units. Finite; must be nonzero and share ``delta_x``'s + sign (see ``on_wrong_sign``). + sigma : float or sequence of float + Standard error of ``delta_y`` (finite, positive). + dims : mapping or sequence of mappings, optional + Extra model-dimension columns, e.g. ``{"geo": "US-CA"}`` for a geo-level MMM + built with ``MMM(dims=("geo",))``. Values must match the target model's + coordinate values exactly and identify the population ``delta_y`` describes + (do not label a multi-geo average with one specific geo). All rows must share + one key set; column order follows the first mapping. + on_wrong_sign : {"raise", "drop", "keep"}, default "raise" + Policy for rows PyMC-Marketing cannot use: ``sign(delta_y)`` contradicting + ``sign(delta_x)`` (rejected upstream with ``NonMonotonicError``), or + ``delta_y == 0`` (degenerate for its strictly-positive Gamma lift likelihood, + which its own monotonicity check does not catch). ``"raise"`` (default) errors + with guidance; ``"drop"`` warns and removes such rows (raises if that would + empty the frame); ``"keep"`` warns and emits them anyway (for non-PyMC + consumers - the frame is not valid PyMC-Marketing input). + + Returns + ------- + pd.DataFrame + Columns ``[channel, *dims, x, delta_x, delta_y, sigma]``, one row per + experiment. + + Raises + ------ + ValueError + On invalid inputs (non-positive ``sigma``, non-finite values, ``delta_x == 0``, + ``x + delta_x < 0``, broadcasting length mismatches, empty inputs, + heterogeneous ``dims`` key sets) or wrong-signed/zero rows under the default + policy. + """ + if on_wrong_sign not in _WRONG_SIGN_POLICIES: + raise ValueError( + f"on_wrong_sign must be one of {_WRONG_SIGN_POLICIES}; got {on_wrong_sign!r}" + ) + + n = _seq_len(channel, x, delta_x, delta_y, sigma, dims) + channels = _broadcast("channel", channel, n) + xs = _broadcast("x", x, n) + delta_xs = _broadcast("delta_x", delta_x, n) + delta_ys = _broadcast("delta_y", delta_y, n) + sigmas = _broadcast("sigma", sigma, n) + dim_cols, dim_rows = _normalize_dims(dims, n) + + rows: List[Dict[str, Any]] = [] + wrong_sign_rows: List[int] = [] + zero_lift_rows: List[int] = [] + for i in range(n): + x_i = float(xs[i]) + dx_i = float(delta_xs[i]) + dy_i = float(delta_ys[i]) + sig_i = _finite_positive("sigma", sigmas[i], i) + if not math.isfinite(x_i) or x_i < 0: + raise ValueError(f"x must be finite and >= 0; got {xs[i]!r} for experiment[{i}]") + if not math.isfinite(dx_i) or dx_i == 0: + raise ValueError( + f"delta_x must be finite and nonzero (the experiment changed spend); " + f"got {delta_xs[i]!r} for experiment[{i}]" + ) + post_spend = x_i + dx_i + if not math.isfinite(post_spend) or post_spend < 0: + raise ValueError( + f"x + delta_x must be finite and >= 0 (post-test spend cannot be " + f"negative or overflow; the saturation curve is evaluated at " + f"x + delta_x); got {x_i!r} + {dx_i!r} = {post_spend!r} for " + f"experiment[{i}]" + ) + if not math.isfinite(dy_i): + raise ValueError(f"delta_y must be finite; got {delta_ys[i]!r} for experiment[{i}]") + if dy_i == 0: + zero_lift_rows.append(i) + elif (dx_i < 0) != (dy_i < 0): + # Compare signs directly: dx_i * dy_i can underflow to -0.0 for tiny + # magnitudes (e.g. 1e-200 * -1e-200), so a product < 0 check misses + # wrong-signed rows. Both are strictly nonzero here (delta_x validated + # above, delta_y != 0 handled just above), so the comparison is exact. + wrong_sign_rows.append(i) + row: Dict[str, Any] = {"channel": channels[i]} + if dim_cols: + row.update({k: dim_rows[i][k] for k in dim_cols}) + row.update({"x": x_i, "delta_x": dx_i, "delta_y": dy_i, "sigma": sig_i}) + rows.append(row) + + # Two invalid-row classes, one shared disposition. Wrong sign is what + # PyMC-Marketing rejects with NonMonotonicError; zero lift passes its + # monotonicity check but is degenerate for the strictly-positive Gamma lift + # likelihood (an insignificant experiment that cannot calibrate saturation). + if wrong_sign_rows or zero_lift_rows: + parts = [] + if wrong_sign_rows: + parts.append( + f"row(s) {wrong_sign_rows} have sign(delta_y) contradicting " + f"sign(delta_x) (PyMC-Marketing rejects these with NonMonotonicError)" + ) + if zero_lift_rows: + parts.append( + f"row(s) {zero_lift_rows} have delta_y == 0 (degenerate for " + f"PyMC-Marketing's strictly-positive Gamma lift likelihood, which its " + f"monotonicity check does not catch)" + ) + detail = "; ".join(parts) + if on_wrong_sign == "raise": + raise ValueError( + f"{detail}. Pool experiments, re-scope, or exclude the offending " + f"experiment; or pass on_wrong_sign='drop'/'keep' to handle these rows " + f"explicitly." + ) + dropped = set(wrong_sign_rows) | set(zero_lift_rows) + if on_wrong_sign == "drop": + if len(dropped) == len(rows): + raise ValueError(f"on_wrong_sign='drop' would remove every row: {detail}.") + warnings.warn( + f"Dropping invalid lift-test row(s): {detail}.", UserWarning, stacklevel=2 + ) + rows = [row for i, row in enumerate(rows) if i not in dropped] + else: # "keep" + warnings.warn( + f"Keeping invalid lift-test row(s): {detail}. The frame is NOT valid " + f"input for PyMC-Marketing's lift likelihood.", + UserWarning, + stacklevel=2, + ) + + columns = ["channel", *dim_cols, "x", "delta_x", "delta_y", "sigma"] + return pd.DataFrame(rows, columns=columns) + + +@dataclass(frozen=True) +class ExperimentROI: + """Per-experiment ROI contribution inside a :class:`MeridianROIPrior`.""" + + roi: float + roi_sd: float + spend: float + weight: float + + def to_dict(self) -> Dict[str, float]: + return { + "roi": self.roi, + "roi_sd": self.roi_sd, + "spend": self.spend, + "weight": self.weight, + } + + +@dataclass(frozen=True) +class MeridianROIPrior: + """Lognormal prior parameters for Meridian's ``roi_m``/``mroi_m`` calibration. + + ``mu``/``sigma`` reproduce ``meridian.model.prior_distribution. + lognormal_dist_from_mean_std(roi_mean, roi_sd)`` exactly (closed form; no Meridian + dependency). ``parameter`` records which Meridian prior the caller is informing + (``"roi_m"`` or ``"mroi_m"``). ``per_experiment`` records each pooled experiment's + ROI, widened sd, spend, and spend weight. + """ + + roi_mean: float + roi_sd: float + mu: float + sigma: float + parameter: str = "roi_m" + per_experiment: Tuple[ExperimentROI, ...] = field(default=()) + + def to_dict(self) -> Dict[str, Any]: + return { + "distribution": "LogNormal", + "parameter": self.parameter, + "roi_mean": self.roi_mean, + "roi_sd": self.roi_sd, + "mu": self.mu, + "sigma": self.sigma, + "per_experiment": [e.to_dict() for e in self.per_experiment], + } + + def to_code( + self, + *, + channel: Optional[str] = None, + media_channels: Optional[Sequence[str]] = None, + single_channel: bool = False, + roi_calibration_period: Optional[str] = None, + full_model_window: bool = False, + ) -> str: + """Ready-to-paste Meridian snippet (channel- and time-scoped; 1.7.0 pinned). + + Meridian's ``roi_m``/``mroi_m`` prior has batch shape ``n_media_channels`` and + a scalar distribution broadcasts to EVERY media channel - a TV experiment's + prior must not silently calibrate search, social, etc. Channel scope is + therefore always explicit: + + - ``to_code(channel="tv", media_channels=["search", "tv"])`` emits vector + ``mu``/``sigma`` in exactly the ``media_channels`` order (which must match + the Meridian ``InputData`` channel order); the experiment channel carries + this prior and every other channel keeps Meridian's default. + - ``to_code(single_channel=True)`` emits the scalar snippet for + single-channel models, marked as such in the generated code. + - Calling with neither raises ``ValueError``. + + The prior's TIME scope is also required: Meridian's default + ``roi_calibration_period=None`` applies the prior over all model times, but + the prior was estimated on the experiment window. Pass + ``roi_calibration_period=""`` (the boolean + ``(n_media_times, n_media_channels)`` mask for your window) or + ``full_model_window=True`` to acknowledge that the two coincide. + + Snippets use the TensorFlow substrate of TensorFlow Probability; JAX-backed + Meridian users should swap the import for + ``tensorflow_probability.substrates.jax`` (noted in the generated code). + """ + if roi_calibration_period is None and not full_model_window: + raise ValueError( + "to_code() needs the prior's time scope: Meridian's default " + "roi_calibration_period=None applies the prior over ALL model times, " + "but this prior was estimated on the EXPERIMENT window, and ROI " + "differs across windows under varying spend and saturation. Pass " + "roi_calibration_period=, " + "or full_model_window=True to acknowledge that the MMM window and the " + "experiment window coincide." + ) + if roi_calibration_period is not None and full_model_window: + raise ValueError( + "pass either roi_calibration_period or full_model_window=True, not both" + ) + if roi_calibration_period is not None: + try: + ast.parse(roi_calibration_period, mode="eval") + except SyntaxError as exc: + raise ValueError( + f"roi_calibration_period must be a valid Python expression building " + f"the boolean (n_media_times, n_media_channels) mask (it is " + f"interpolated verbatim into the snippet); got " + f"{roi_calibration_period!r}, which does not parse: {exc.msg}" + ) from exc + window_note = ( + "Experiment window == full model window (acknowledged via full_model_window=True)." + if full_model_window + else "Mask restricting the prior to the experiment window." + ) + calibration_period = "None" if full_model_window else roi_calibration_period + prior_type = "roi" if self.parameter == "roi_m" else "mroi" + if media_channels is not None: + if single_channel: + raise ValueError("pass either media_channels or single_channel=True, not both") + channels = list(media_channels) + if not channels: + raise ValueError("media_channels must be non-empty") + if channel is None or channel not in channels: + raise ValueError( + f"channel must name the experiment channel within media_channels; " + f"got channel={channel!r}, media_channels={channels!r}" + ) + default_mu, default_sigma = _MERIDIAN_PARAM_DEFAULTS[self.parameter] + mu_vector = [self.mu if c == channel else default_mu for c in channels] + sigma_vector = [self.sigma if c == channel else default_sigma for c in channels] + return _MERIDIAN_MULTI_CHANNEL_TEMPLATE.format( + channels=channels, + channel=channel, + mu_vector=mu_vector, + sigma_vector=sigma_vector, + param=self.parameter, + default_mu=default_mu, + default_sigma=default_sigma, + window_note=window_note, + calibration_period=calibration_period, + prior_type=prior_type, + ) + if single_channel: + return _MERIDIAN_SINGLE_CHANNEL_TEMPLATE.format( + mu=self.mu, + sigma=self.sigma, + param=self.parameter, + window_note=window_note, + calibration_period=calibration_period, + prior_type=prior_type, + ) + raise ValueError( + "to_code() needs explicit channel scope: a scalar prior broadcasts to " + "every media channel in a multi-channel Meridian model. Pass channel=... " + "with media_channels=[...] (vector prior in model channel order), or " + "single_channel=True for a single-channel model." + ) + + +def to_meridian_roi_prior( + *, + incremental_outcome: Union[float, Sequence[float]], + incremental_outcome_se: Union[float, Sequence[float]], + spend: Union[float, Sequence[float]], + parameter: str = "roi_m", + se_widening: float = 1.0, +) -> MeridianROIPrior: + """Build a Meridian lognormal ROI/mROI prior from scoped experiment result(s). + + Per experiment ``roi = incremental_outcome / spend`` with standard deviation + ``incremental_outcome_se / spend * se_widening``. Multiple experiments for the same + channel are pooled with spend weights - the spend-weighted average ROI the Meridian + FAQ suggests for multiple experiments (citing section 3.4 of Google's MMM + calibration whitepaper): ``roi_mean = sum(w_i * roi_i)`` with + ``w_i = spend_i / sum(spend)``. The pooled ``roi_sd = sqrt(sum((w_i * sd_i)^2))`` + is this library's uncertainty propagation for that average (treats experiments as + independent - widen via ``se_widening`` when experiments share controls or windows). + The pooled mean/sd map to lognormal ``(mu, sigma)`` via Google's closed form. + + **The caller supplies the scoped estimand.** ``incremental_outcome`` is the total + incremental outcome the experiment measured, matching the estimand of the target + prior: for ``parameter="roi_m"`` that is the outcome attributable to the channel's + spend against a zero-spend counterfactual (a full holdout), divided here by + ``spend`` = the channel's total spend over the window; for ``parameter="mroi_m"`` + it is the marginal outcome of the spend change, divided by that spend change. Sign, + aggregation, and population are the caller's responsibility - diff-diff does not + rescale a headline ATT. + + Parameters + ---------- + incremental_outcome : float or sequence of float + Total incremental outcome per experiment (the estimand matching ``parameter``), + in the same currency/units as the MMM. Must be finite; the pooled ROI must be + positive (lognormal support). + incremental_outcome_se : float or sequence of float + Standard error of ``incremental_outcome`` (finite, positive), computed for the + caller's aggregation. + spend : float or sequence of float + Spend the incremental outcome is divided by (finite, positive): the channel's + total spend over the window for ``roi_m``, or the spend change for ``mroi_m``. + parameter : {"roi_m", "mroi_m"}, default "roi_m" + Which Meridian prior the estimate informs. ``roi_m`` is the return on the + channel's full spend (a full-holdout / zero-spend estimand); ``mroi_m`` is the + marginal return of a spend change. Under saturation these differ, so the caller + declares which their ``incremental_outcome`` measures; the returned prior and + ``.to_code()`` target that parameter with its own Meridian default for + non-experiment channels. + se_widening : float, default 1.0 + Multiplier on each experiment's ROI standard deviation (finite, positive). + Values above 1 encode skepticism about experiment-to-MMM transferability. + + Returns + ------- + MeridianROIPrior + Pooled ``roi_mean``/``roi_sd``, the lognormal ``mu``/``sigma``, the target + ``parameter``, per-experiment detail, plus ``.to_dict()`` and the + channel-scoped ``.to_code()`` snippet helper. + + Raises + ------ + ValueError + On an invalid ``parameter``, non-positive ``spend``/``se``/``se_widening``, + non-finite inputs, broadcasting mismatches, empty inputs, a non-positive pooled + ROI mean (lognormal support), or non-finite scaled/pooled results. + """ + if parameter not in _MERIDIAN_PARAM_DEFAULTS: + raise ValueError( + f"parameter must be one of {sorted(_MERIDIAN_PARAM_DEFAULTS)}; got " + f"{parameter!r}. roi_m is the return on the channel's full spend (a " + f"full-holdout / zero-spend estimand); mroi_m is the marginal return of a " + f"spend change. Under saturation they differ - pass the one your " + f"incremental_outcome measures." + ) + se_w = float(se_widening) + if not math.isfinite(se_w) or se_w <= 0: + raise ValueError(f"se_widening must be finite and > 0; got {se_widening!r}") + + n = _seq_len(incremental_outcome, incremental_outcome_se, spend) + outcomes = _broadcast("incremental_outcome", incremental_outcome, n) + outcome_ses = _broadcast("incremental_outcome_se", incremental_outcome_se, n) + spends = _broadcast("spend", spend, n) + + rois: List[float] = [] + sds: List[float] = [] + spend_vals: List[float] = [] + for i in range(n): + total = float(outcomes[i]) + if not math.isfinite(total): + raise ValueError( + f"incremental_outcome must be finite; got {outcomes[i]!r} for experiment[{i}]" + ) + total_se = _finite_positive("incremental_outcome_se", outcome_ses[i], i) + spend_i = _finite_positive("spend", spends[i], i) + roi_i = total / spend_i + sd_i = total_se / spend_i * se_w + if not (math.isfinite(roi_i) and math.isfinite(sd_i) and sd_i > 0): + raise ValueError( + f"experiment[{i}] ROI is not finite-positive (roi={roi_i!r}, " + f"roi_sd={sd_i!r}); the magnitudes overflowed or underflowed the float " + f"range - rescale the outcome or spend units" + ) + rois.append(roi_i) + sds.append(sd_i) + spend_vals.append(spend_i) + + total_spend = sum(spend_vals) + weights = [s / total_spend for s in spend_vals] + roi_mean = sum(w * r for w, r in zip(weights, rois)) + # math.hypot is a scaled Euclidean norm: it avoids the intermediate squaring + # that would raise a raw OverflowError before the finiteness guard below. + roi_sd = math.hypot(*(w * s for w, s in zip(weights, sds))) + + if not (math.isfinite(roi_mean) and math.isfinite(roi_sd) and roi_sd > 0): + raise ValueError( + f"pooled ROI moments are not finite-positive (roi_mean={roi_mean!r}, " + f"roi_sd={roi_sd!r}); the per-experiment magnitudes overflowed or " + f"underflowed - rescale the units" + ) + if roi_mean <= 0: + raise ValueError( + f"Pooled ROI mean is {roi_mean:.6g} <= 0, which cannot map to Meridian's " + f"lognormal {parameter} prior (strictly positive support). Options: pool " + f"with additional experiments, use a wider prior from a defensible positive " + f"range, or calibrate via Meridian's contribution/coefficient " + f"parameterizations manually." + ) + + # log_term = log1p((roi_sd / roi_mean)**2), computed in the log domain so an + # extreme coefficient of variation cannot overflow the squaring before the + # finiteness guard: logaddexp(0, 2*ln(sd/mean)) == log(1 + (sd/mean)**2), and + # it also keeps a tiny relative SE from rounding sigma to 0. + log_term = float(np.logaddexp(0.0, 2.0 * (math.log(roi_sd) - math.log(roi_mean)))) + sigma = math.sqrt(log_term) + mu = math.log(roi_mean) - 0.5 * log_term + if not (math.isfinite(mu) and math.isfinite(sigma) and sigma > 0): + raise ValueError( + f"lognormal parameters are not finite (mu={mu!r}, sigma={sigma!r}) - the " + f"ROI mean/sd ratio is outside the representable range; rescale the outcome " + f"or spend units and re-export" + ) + + per_experiment = tuple( + ExperimentROI(roi=r, roi_sd=s, spend=sp, weight=w) + for r, s, sp, w in zip(rois, sds, spend_vals, weights) + ) + return MeridianROIPrior( + roi_mean=roi_mean, + roi_sd=roi_sd, + mu=mu, + sigma=sigma, + parameter=parameter, + per_experiment=per_experiment, + ) diff --git a/docs/api/index.rst b/docs/api/index.rst index 6301c5449..6393c4849 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -246,6 +246,20 @@ result objects: diff_diff.DiagnosticReport diff_diff.DiagnosticReportResults +MMM Calibration Export +---------------------- + +Convert experiment results into Marketing Mix Model calibration inputs +(PyMC-Marketing lift tests, Google Meridian ROI priors): + +.. autosummary:: + :toctree: _autosummary + :nosignatures: + + diff_diff.to_pymc_marketing_lift_test + diff_diff.to_meridian_roi_prior + diff_diff.MeridianROIPrior + Boundary Local-Linear Estimators -------------------------------- @@ -370,6 +384,7 @@ Reporting business_report diagnostic_report + mmm Results & Visualization ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/api/mmm.rst b/docs/api/mmm.rst new file mode 100644 index 000000000..615f856a2 --- /dev/null +++ b/docs/api/mmm.rst @@ -0,0 +1,124 @@ +MMM Calibration Export +====================== + +Assemble Marketing Mix Model (MMM) calibration inputs from experiment results. + +.. module:: diff_diff.mmm + +Overview +-------- + +MMM practitioners calibrate their models against experimental evidence. The two +dominant Python MMM frameworks consume that evidence in different shapes: + +1. **PyMC-Marketing** (and prophetverse, which mirrors the same schema) ingests a + lift-test DataFrame - columns ``channel``, optional model dims (e.g. ``geo``), + ``x``, ``delta_x``, ``delta_y``, ``sigma`` - via + ``MMM.add_lift_test_measurements``. + +2. **Google Meridian** calibrates through lognormal priors: ``roi_m`` for the return + on a channel's full spend (a zero-spend / full-holdout estimand) and ``mroi_m`` + for a marginal return. The point estimate maps to the prior mean and its standard + error to the prior standard deviation, converted to lognormal ``(mu, sigma)`` with + Google's closed form. + +**Explicit in, validated out.** These functions do not rescale a result's headline +ATT. Reconciling an experiment's estimate to a calibration input needs context +diff-diff cannot see - the target MMM's row granularity (per-geo vs national), its +time window, and the outcome's scale (additive levels vs a log/rate/share) - so the +caller supplies the already-scoped incremental outcome and its standard error (the +numbers read off a fitted result's ``summary()``, aggregated to the population and +window one MMM row represents). diff-diff does only what it can verify: assemble the +exact schema, enforce each consumer's guards, convert to the lognormal +parameterization (parity with Google's closed form), pool, and emit snippets. It is +pure numpy/pandas and imports no MMM package. + +to_pymc_marketing_lift_test +--------------------------- + +Assemble a lift-test DataFrame for ``pymc_marketing.mmm.MMM.add_lift_test_measurements``. + +.. autofunction:: diff_diff.to_pymc_marketing_lift_test + +Example +~~~~~~~ + +.. code-block:: python + + from diff_diff import SyntheticDiD, to_pymc_marketing_lift_test + + # Single-treated-geo experiment (US-CA): TV spend raised there vs control + # geos. With exactly ONE treated geo, the SDID ATT is that geo's per-week + # lift, so labelling the row with its coordinate is sound. (For MULTIPLE + # treated geos the pooled ATT is an average - do not assign it to one geo; + # export each geo's own effect, or omit the geo dim and match aggregate + # spend to an aggregate lift.) + result = SyntheticDiD().fit(panel, outcome='revenue', treatment='treated', + unit='geo', time='week') + + df_lift = to_pymc_marketing_lift_test( + channel='tv', + x=50_000.0, # baseline weekly TV spend in US-CA + delta_x=20_000.0, # spend increase during the test + delta_y=result.att, # US-CA's per-week lift (one treated geo) + sigma=result.se, # its standard error + dims={'geo': 'US-CA'}, + ) + # -> columns [channel, geo, x, delta_x, delta_y, sigma], ready for + # MMM.add_lift_test_measurements(df_lift) + +to_meridian_roi_prior +--------------------- + +Build a Meridian lognormal ``roi_m``/``mroi_m`` prior from scoped experiment result(s). + +.. autofunction:: diff_diff.to_meridian_roi_prior + +Example +~~~~~~~ + +.. code-block:: python + + from diff_diff import DifferenceInDifferences, to_meridian_roi_prior + + result = DifferenceInDifferences().fit(panel, outcome='revenue', + treatment='treated', time='post') + + # Caller aggregates the ATT to a total incremental outcome over the treated + # population and window (e.g. att x treated units x post periods) and supplies + # its SE - diff-diff does not rescale the headline effect. + prior = to_meridian_roi_prior( + incremental_outcome=180_000.0, # total incremental revenue + incremental_outcome_se=45_000.0, # its standard error + spend=200_000.0, # total channel spend over the window + parameter='roi_m', # or 'mroi_m' for a marginal experiment + se_widening=1.5, # optional transferability skepticism + ) + print(prior.roi_mean, prior.roi_sd) + + # Channel- and time-scoped snippet: roi_m is per-channel, and the prior was + # estimated on the experiment window. + print(prior.to_code(channel='tv', media_channels=['search', 'tv'], + roi_calibration_period='experiment_window_mask')) + +MeridianROIPrior +---------------- + +Result container for the Meridian exporter. + +.. autoclass:: diff_diff.MeridianROIPrior + :no-index: + :members: + :undoc-members: + +References +---------- + +- PyMC-Marketing lift-test calibration: + https://www.pymc-marketing.io/en/stable/notebooks/mmm/mmm_lift_test.html +- Google Meridian, "Set custom prior distributions using past experiments": + https://developers.google.com/meridian/docs/advanced-modeling/set-custom-priors-past-experiments +- Google Meridian, ROI/mROI/contribution parameterizations: + https://developers.google.com/meridian/docs/advanced-modeling/roi-mroi-contribution-parameterizations +- Zhou, G., Choe, Y., & Hetrakul, C. (2023). Calibrated MMM better predicts true + ROAS. Meta Marketing Science. diff --git a/docs/doc-deps.yaml b/docs/doc-deps.yaml index 36554e505..81fdfe67d 100644 --- a/docs/doc-deps.yaml +++ b/docs/doc-deps.yaml @@ -908,6 +908,27 @@ sources: section: "DiagnosticReport" type: user_guide + diff_diff/mmm.py: + drift_risk: medium + docs: + - path: docs/methodology/REGISTRY.md + section: "MMM Calibration Export (interop)" + type: methodology + note: "Lift-test scaling semantics, spend-weighted pooling, lognormal closed form (pinned to Meridian 1.7.0), sign/positivity policies." + - path: docs/api/mmm.rst + type: api_reference + - path: README.md + section: "Diagnostics & Sensitivity (one-line mention)" + type: user_guide + - path: docs/references.rst + type: user_guide + - path: diff_diff/guides/llms.txt + section: "Diagnostics and Sensitivity Analysis" + type: user_guide + - path: diff_diff/guides/llms-full.txt + section: "MMM Calibration Export" + type: user_guide + diff_diff/power.py: drift_risk: low docs: diff --git a/docs/methodology/REGISTRY.md b/docs/methodology/REGISTRY.md index 50bfc7f4c..6ae00ce8e 100644 --- a/docs/methodology/REGISTRY.md +++ b/docs/methodology/REGISTRY.md @@ -5768,6 +5768,33 @@ estimator-focused: --- +## MMM Calibration Export (interop) + +**Primary sources:** [PyMC-Marketing lift-test calibration](https://www.pymc-marketing.io/en/stable/notebooks/mmm/mmm_lift_test.html) (`MMM.add_lift_test_measurements` schema); [Google Meridian, "Set custom prior distributions using past experiments"](https://developers.google.com/meridian/docs/advanced-modeling/set-custom-priors-past-experiments) and the closed-form lognormal conversion in `meridian/model/prior_distribution.py` (`lognormal_dist_from_mean_std`, Meridian 1.7.0); the roi_m/mroi_m estimand definitions in [Meridian's ROI parameterization docs](https://developers.google.com/meridian/docs/advanced-modeling/roi-mroi-contribution-parameterizations). + +**Module:** `diff_diff/mmm.py` + +**Scope:** Interop builders (not estimators), **explicit in / validated out**. Reconciling an experiment's estimate to a calibration input requires context diff-diff cannot see - the target MMM's row granularity (per-geo vs national), its time window, and the outcome's scale (additive levels vs log/rate/share) - so the module does NOT rescale a result's headline ATT. The caller supplies the already-scoped incremental outcome and its SE (the numbers read off `summary()`, aggregated to the population and window their MMM row represents); diff-diff assembles the target schema, enforces each consumer's guards, converts to the lognormal parameterization, pools, and emits snippets. Pure numpy/pandas; no MMM package imported; no result introspection, so the module is purely additive (does not touch any estimator). Deriving totals from a fitted result is deferred to the post-4.0 `results.aggregate()` layer, where the estimator - which owns its aggregation weights, balance, and survey masses - can compute its own total effect and SE. + +**Key implementation requirements:** + +*Lift-test frame (`to_pymc_marketing_lift_test`):* +- Assembles one row per experiment with the exact schema `[channel, *dims, x, delta_x, delta_y, sigma]` from caller-supplied values. `delta_y`/`sigma` are the scoped incremental outcome and its SE; all four numeric columns must describe the SAME calibration observation (same channel, population, period span, additive-level outcome) - stated in the docstring, not machine-checkable, since spend and scope are user input. +- Sign/zero policy (`on_wrong_sign={raise,drop,keep}`, default raise): PyMC-Marketing rejects `sign(delta_y) != sign(delta_x)` with `NonMonotonicError`; `delta_y == 0` is a distinct class (degenerate for its strictly-positive Gamma lift likelihood, which its own monotonicity check does not catch). Both share one disposition; `drop` raises rather than emit an empty frame; `keep` warns the frame is not valid PyMC-Marketing input. +- Guards: `sigma > 0` and finite; `delta_x` finite and nonzero; `x >= 0` finite; `x + delta_x >= 0` (post-test spend cannot be negative - the saturation curve is evaluated there); `delta_y` finite; go-dark tests (`delta_x < 0` with sign-consistent negative lift) pass. `dims` keys must not collide with reserved schema columns and must share one key set across rows; non-mapping dims raise `TypeError`. + +*Meridian ROI prior (`to_meridian_roi_prior`):* +- Per experiment `roi = incremental_outcome / spend`, `roi_sd = incremental_outcome_se / spend * se_widening`. Multiple experiments pooled spend-weighted - the spend-weighted average ROI the [Meridian FAQ](https://developers.google.com/meridian/docs/faqs) suggests (citing sec 3.4 of Google's MMM calibration whitepaper): `roi_mean = sum(w_i * roi_i)`, `w_i = spend_i / sum(spend)`. +- **Estimand (`parameter={roi_m,mroi_m}`, default roi_m):** `roi_m` is the return on the channel's full spend (a zero-spend/full-holdout estimand; `spend` = total channel spend); `mroi_m` is the marginal return of a spend change (`spend` = the spend change). Under saturation these differ, so the caller declares which their `incremental_outcome` measures; the returned prior and `.to_code()` target that parameter with its own Meridian default for non-experiment channels (roi_m `LogNormal(0.2, 0.9)`, mroi_m `LogNormal(0.0, 0.5)`, both verified at 1.7.0). Sign, aggregation, and population are the caller's responsibility - a go-dark contract's numerator is `Y_exposed - Y_zero`, which the caller supplies with the correct sign. +- Lognormal conversion matches Meridian's `lognormal_dist_from_mean_std` exactly: `sigma = sqrt(log1p((s/m)^2))`, `mu = ln(m) - log1p((s/m)^2)/2` (`log1p` keeps a ~1e-8 relative SE from rounding `sigma` to 0). +- A non-positive pooled ROI mean raises (lognormal positivity), pointing to pooling, wider priors, or Meridian's contribution/coefficient parameterizations. `spend`, `incremental_outcome_se`, and `se_widening` must be finite and positive; `incremental_outcome` finite. +- **Note:** Pooled `roi_sd = sqrt(sum((w_i * sd_i)^2))` treats experiments as independent (no covariance term). Experiments sharing control units or overlapping windows are positively correlated and the pooled sd is then anti-conservative; the docstring instructs users to widen via `se_widening`. +- **Channel scope in `.to_code()`**: roi_m/mroi_m have batch shape `n_media_channels` and a scalar distribution broadcasts to EVERY channel, so the snippet helper requires explicit scope - `channel=`+`media_channels=` emits a vector prior in model channel order (non-experiment channels keep the parameter's Meridian default), `single_channel=True` emits a scalar snippet marked single-channel-only, and neither raises. It also requires the prior's TIME scope (`roi_calibration_period=` or `full_model_window=True`), because Meridian's default applies an experiment-window prior over all model times; and it sets `media_prior_type="roi"`/`"mroi"` on `ModelSpec` (Meridian ignores a supplied `roi_m`/`mroi_m` unless the matching prior type is selected). Snippets use the TensorFlow substrate; a comment points JAX users at `tensorflow_probability.substrates.jax`. + +*Outputs:* All scaled and pooled values (`roi`, `roi_sd`, pooled moments, lognormal `mu`/`sigma`) are validated finite-and-positive after arithmetic; overflow/underflow raises rather than emitting non-finite or zero-uncertainty calibration data. + +--- + # Version History - **v1.3** (2026-03-26): Added Replicate Weight Variance, DEFF Diagnostics, diff --git a/docs/references.rst b/docs/references.rst index 478b9c0c9..80082986b 100644 --- a/docs/references.rst +++ b/docs/references.rst @@ -350,6 +350,21 @@ Power Analysis Establishes the power gains from multiple pre/post periods in panel experiments (the ``(m+r)/(mr)`` period factor); the ψ = 0 special case of Burlig, Preonas & Woerman (2020) Eq. 2. +MMM Calibration Interop +----------------------- + +- **PyMC Labs.** "Lift Test Calibration." *PyMC-Marketing documentation*. https://www.pymc-marketing.io/en/stable/notebooks/mmm/mmm_lift_test.html + + Defines the lift-test DataFrame schema (``channel``/dims/``x``/``delta_x``/``delta_y``/``sigma``) and the saturation-curve likelihood consumed by ``MMM.add_lift_test_measurements``, which ``to_pymc_marketing_lift_test`` targets. + +- **Google.** "Set Custom Prior Distributions Using Past Experiments." *Google Meridian documentation*. https://developers.google.com/meridian/docs/advanced-modeling/set-custom-priors-past-experiments + + Documents the experiment-to-ROI-prior calibration workflow and caveats; the lognormal ``(mu, sigma)`` closed form emitted by ``to_meridian_roi_prior`` matches Meridian's ``prior_distribution.lognormal_dist_from_mean_std`` (verified against Meridian 1.7.0 source). + +- **Zhou, G., Choe, Y., & Hetrakul, C. (2023).** "Calibrated MMM Better Predicts True ROAS." *Meta Marketing Science*. https://medium.com/@gufengzhou/calibrated-mmm-better-predicts-true-roas-d5adfc8abdc4 + + Empirical motivation for experiment calibration: calibrating MMMs against lift experiments substantially reduces ROAS prediction error. + General Causal Inference ------------------------ diff --git a/tests/test_mmm.py b/tests/test_mmm.py new file mode 100644 index 000000000..dc3e73fab --- /dev/null +++ b/tests/test_mmm.py @@ -0,0 +1,471 @@ +"""Tests for diff_diff.mmm - MMM calibration input assembly (interop). + +The exporters are explicit-in / validated-out: the caller supplies the already-scoped +incremental outcome and its SE, and the module assembles the target schema, enforces +each consumer's guards, converts to the lognormal parameterization, and pools. These +tests cover the schema/guards/math directly, plus a realistic workflow test that reads +att/se off fitted estimators and feeds them in (the module never introspects results). +""" + +import math +import warnings + +import pytest +from scipy import stats + +from diff_diff import ( + CallawaySantAnna, + DifferenceInDifferences, + to_meridian_roi_prior, + to_pymc_marketing_lift_test, +) +from diff_diff.mmm import MeridianROIPrior +from diff_diff.prep import generate_did_data, generate_staggered_data + + +class TestLiftTestFrame: + def test_columns_and_passthrough(self): + df = to_pymc_marketing_lift_test( + channel="tv", x=50_000.0, delta_x=20_000.0, delta_y=120.0, sigma=30.0 + ) + assert list(df.columns) == ["channel", "x", "delta_x", "delta_y", "sigma"] + row = df.iloc[0] + assert (row["channel"], row["x"], row["delta_x"], row["delta_y"], row["sigma"]) == ( + "tv", + 50_000.0, + 20_000.0, + 120.0, + 30.0, + ) + + def test_multiple_experiments_broadcast(self): + df = to_pymc_marketing_lift_test( + channel="tv", + x=[100.0, 200.0], + delta_x=50.0, + delta_y=[10.0, 20.0], + sigma=[2.0, 4.0], + ) + assert list(df["x"]) == [100.0, 200.0] + assert list(df["delta_x"]) == [50.0, 50.0] + assert list(df["delta_y"]) == [10.0, 20.0] + + def test_dims_columns_between_channel_and_x(self): + df = to_pymc_marketing_lift_test( + channel="search", + x=[100.0, 200.0], + delta_x=50.0, + delta_y=[10.0, 20.0], + sigma=[2.0, 4.0], + dims=[{"geo": "US-CA"}, {"geo": "US-NY"}], + ) + assert list(df.columns) == ["channel", "geo", "x", "delta_x", "delta_y", "sigma"] + assert list(df["geo"]) == ["US-CA", "US-NY"] + + def test_single_dims_mapping_broadcasts(self): + df = to_pymc_marketing_lift_test( + channel="tv", + x=[1.0, 2.0], + delta_x=1.0, + delta_y=[1.0, 1.0], + sigma=1.0, + dims={"geo": "US-CA"}, + ) + assert list(df["geo"]) == ["US-CA", "US-CA"] + + def test_dims_non_mapping_raises(self): + with pytest.raises(TypeError, match="sequence of mappings"): + to_pymc_marketing_lift_test( + channel="tv", x=1.0, delta_x=1.0, delta_y=1.0, sigma=1.0, dims=["geo"] + ) + + def test_dims_reserved_collision_raises(self): + with pytest.raises(ValueError, match="reserved lift-test columns"): + to_pymc_marketing_lift_test( + channel="tv", + x=1.0, + delta_x=1.0, + delta_y=1.0, + sigma=1.0, + dims={"sigma": "s"}, + ) + + def test_heterogeneous_dims_keys_raise(self): + with pytest.raises(ValueError, match="share one key set"): + to_pymc_marketing_lift_test( + channel="tv", + x=1.0, + delta_x=1.0, + delta_y=[1.0, 1.0], + sigma=1.0, + dims=[{"geo": "US-CA"}, {"region": "NY"}], + ) + + def test_length_mismatch_raises(self): + with pytest.raises(ValueError, match="delta_y has length 2 but 3"): + to_pymc_marketing_lift_test( + channel="tv", x=[1.0, 2.0, 3.0], delta_x=1.0, delta_y=[1.0, 2.0], sigma=1.0 + ) + + def test_empty_sequence_raises(self): + with pytest.raises(ValueError, match="empty sequence"): + to_pymc_marketing_lift_test(channel="tv", x=[], delta_x=1.0, delta_y=1.0, sigma=1.0) + + def test_sigma_must_be_positive(self): + for bad in (0.0, -1.0, float("nan")): + with pytest.raises(ValueError, match="sigma must be finite and > 0"): + to_pymc_marketing_lift_test( + channel="tv", x=1.0, delta_x=1.0, delta_y=1.0, sigma=bad + ) + + def test_delta_x_zero_raises(self): + with pytest.raises(ValueError, match="delta_x must be finite and nonzero"): + to_pymc_marketing_lift_test(channel="tv", x=1.0, delta_x=0.0, delta_y=1.0, sigma=1.0) + + def test_negative_x_raises(self): + with pytest.raises(ValueError, match="x must be finite"): + to_pymc_marketing_lift_test(channel="tv", x=-5.0, delta_x=1.0, delta_y=1.0, sigma=1.0) + + def test_negative_post_test_spend_raises(self): + with pytest.raises(ValueError, match="x \\+ delta_x must be finite and >= 0"): + to_pymc_marketing_lift_test( + channel="tv", x=100.0, delta_x=-101.0, delta_y=-40.0, sigma=10.0 + ) + + def test_non_finite_delta_y_raises(self): + with pytest.raises(ValueError, match="delta_y must be finite"): + to_pymc_marketing_lift_test( + channel="tv", x=1.0, delta_x=1.0, delta_y=float("nan"), sigma=1.0 + ) + + def test_go_dark_is_valid(self): + with warnings.catch_warnings(): + warnings.simplefilter("error") + df = to_pymc_marketing_lift_test( + channel="tv", x=1_000.0, delta_x=-500.0, delta_y=-40.0, sigma=10.0 + ) + assert df.loc[0, "delta_y"] == -40.0 + + def test_go_dark_to_zero_spend_is_valid(self): + df = to_pymc_marketing_lift_test( + channel="tv", x=1_000.0, delta_x=-1_000.0, delta_y=-40.0, sigma=10.0 + ) + assert df.loc[0, "delta_x"] == -1_000.0 + + def test_wrong_sign_raises_by_default(self): + with pytest.raises(ValueError, match="NonMonotonicError"): + to_pymc_marketing_lift_test( + channel="tv", x=1_000.0, delta_x=500.0, delta_y=-40.0, sigma=10.0 + ) + + def test_wrong_sign_drop_and_keep(self): + with pytest.warns(UserWarning, match="Dropping invalid lift-test"): + df = to_pymc_marketing_lift_test( + channel="tv", + x=1_000.0, + delta_x=500.0, + delta_y=[10.0, -40.0], + sigma=[2.0, 10.0], + on_wrong_sign="drop", + ) + assert len(df) == 1 and df.loc[0, "delta_y"] == 10.0 + with pytest.warns(UserWarning, match="Keeping invalid lift-test"): + df2 = to_pymc_marketing_lift_test( + channel="tv", + x=1_000.0, + delta_x=500.0, + delta_y=-40.0, + sigma=10.0, + on_wrong_sign="keep", + ) + assert df2.loc[0, "delta_y"] == -40.0 + + def test_wrong_sign_drop_all_raises(self): + with pytest.raises(ValueError, match="would remove every row"): + to_pymc_marketing_lift_test( + channel="tv", + x=1_000.0, + delta_x=500.0, + delta_y=-40.0, + sigma=10.0, + on_wrong_sign="drop", + ) + + def test_tiny_wrong_sign_detected_despite_underflow(self): + # delta_x * delta_y underflows to -0.0 here; a direct sign comparison + # still catches the wrong sign. + with pytest.raises(ValueError, match="NonMonotonicError"): + to_pymc_marketing_lift_test( + channel="tv", x=1.0, delta_x=1e-200, delta_y=-1e-200, sigma=1.0 + ) + + def test_zero_lift_names_gamma(self): + with pytest.raises(ValueError, match="Gamma lift likelihood"): + to_pymc_marketing_lift_test( + channel="tv", x=1_000.0, delta_x=500.0, delta_y=0.0, sigma=10.0 + ) + + def test_zero_lift_drop_removes_only_zero(self): + with pytest.warns(UserWarning, match="delta_y == 0"): + df = to_pymc_marketing_lift_test( + channel="tv", + x=1_000.0, + delta_x=500.0, + delta_y=[10.0, 0.0], + sigma=[2.0, 2.0], + on_wrong_sign="drop", + ) + assert len(df) == 1 and df.loc[0, "delta_y"] == 10.0 + + def test_invalid_policy_value(self): + with pytest.raises(ValueError, match="on_wrong_sign must be"): + to_pymc_marketing_lift_test( + channel="tv", x=1.0, delta_x=1.0, delta_y=1.0, sigma=1.0, on_wrong_sign="x" + ) + + def test_post_spend_overflow_raises(self): + # x + delta_x must be finite, not just non-negative. + with pytest.raises(ValueError, match="finite and >= 0"): + to_pymc_marketing_lift_test( + channel="tv", x=1e308, delta_x=1e308, delta_y=1.0, sigma=1.0 + ) + + +class TestMeridianROIPrior: + def test_roi_math_and_lognormal_roundtrip(self): + prior = to_meridian_roi_prior( + incremental_outcome=9_600.0, incremental_outcome_se=2_400.0, spend=20_000.0 + ) + assert prior.roi_mean == pytest.approx(9_600.0 / 20_000.0) + assert prior.roi_sd == pytest.approx(2_400.0 / 20_000.0) + assert prior.parameter == "roi_m" + dist = stats.lognorm(s=prior.sigma, scale=math.exp(prior.mu)) + assert dist.mean() == pytest.approx(prior.roi_mean, rel=1e-12) + assert dist.std() == pytest.approx(prior.roi_sd, rel=1e-12) + + def test_spend_weighted_pooling(self): + prior = to_meridian_roi_prior( + incremental_outcome=[9_600.0, 9_600.0], + incremental_outcome_se=[2_400.0, 2_400.0], + spend=[20_000.0, 30_000.0], + ) + roi_1, roi_2 = 9_600.0 / 20_000.0, 9_600.0 / 30_000.0 + w_1, w_2 = 0.4, 0.6 + assert prior.roi_mean == pytest.approx(w_1 * roi_1 + w_2 * roi_2) + sd_1, sd_2 = 2_400.0 / 20_000.0, 2_400.0 / 30_000.0 + assert prior.roi_sd == pytest.approx(math.hypot(w_1 * sd_1, w_2 * sd_2)) + assert [e.weight for e in prior.per_experiment] == pytest.approx([w_1, w_2]) + + def test_se_widening_scales_sd_only(self): + base = to_meridian_roi_prior( + incremental_outcome=9_600.0, incremental_outcome_se=2_400.0, spend=20_000.0 + ) + wide = to_meridian_roi_prior( + incremental_outcome=9_600.0, + incremental_outcome_se=2_400.0, + spend=20_000.0, + se_widening=2.0, + ) + assert wide.roi_mean == pytest.approx(base.roi_mean) + assert wide.roi_sd == pytest.approx(2.0 * base.roi_sd) + + def test_mroi_m_parameter(self): + prior = to_meridian_roi_prior( + incremental_outcome=500.0, + incremental_outcome_se=100.0, + spend=1_000.0, + parameter="mroi_m", + ) + assert prior.parameter == "mroi_m" + assert prior.to_dict()["parameter"] == "mroi_m" + + def test_invalid_parameter_raises(self): + with pytest.raises(ValueError, match="parameter must be one of"): + to_meridian_roi_prior( + incremental_outcome=1.0, + incremental_outcome_se=1.0, + spend=1.0, + parameter="contribution", + ) + + def test_spend_and_se_must_be_positive(self): + with pytest.raises(ValueError, match="spend must be finite and > 0"): + to_meridian_roi_prior(incremental_outcome=1.0, incremental_outcome_se=1.0, spend=0.0) + with pytest.raises(ValueError, match="incremental_outcome_se must be finite and > 0"): + to_meridian_roi_prior(incremental_outcome=1.0, incremental_outcome_se=-1.0, spend=1.0) + + def test_se_widening_invalid_raises(self): + for bad in (0.0, -1.0, float("inf")): + with pytest.raises(ValueError, match="se_widening"): + to_meridian_roi_prior( + incremental_outcome=1.0, + incremental_outcome_se=1.0, + spend=1.0, + se_widening=bad, + ) + + def test_non_positive_pooled_roi_raises(self): + with pytest.raises(ValueError, match="positive support"): + to_meridian_roi_prior( + incremental_outcome=-500.0, incremental_outcome_se=100.0, spend=1_000.0 + ) + + def test_overflow_raises_not_inf(self): + with pytest.raises(ValueError, match="finite-positive"): + to_meridian_roi_prior( + incremental_outcome=1e308, incremental_outcome_se=1e307, spend=1e-10 + ) + + def test_small_relative_se_keeps_positive_sigma(self): + prior = to_meridian_roi_prior( + incremental_outcome=1.0, incremental_outcome_se=1e-8, spend=1.0 + ) + assert prior.sigma > 0 and math.isfinite(prior.mu) + + def test_extreme_coefficient_of_variation_no_overflow(self): + # A huge roi_sd/roi_mean ratio would overflow (sd/mean)**2; the log-domain + # log1p keeps mu/sigma finite instead of raising a raw OverflowError. + prior = to_meridian_roi_prior( + incremental_outcome=1e-300, incremental_outcome_se=1.0, spend=1.0 + ) + assert math.isfinite(prior.mu) and math.isfinite(prior.sigma) and prior.sigma > 0 + + def test_pooled_sd_uses_scaled_norm_no_overflow(self): + # Per-experiment SDs whose naive sum-of-squares would overflow (>1.8e308) + # but whose true weighted norm is finite: math.hypot keeps it finite + # instead of raising a raw OverflowError. + prior = to_meridian_roi_prior( + incremental_outcome=[1e160, 1e160], + incremental_outcome_se=[1e160, 1e160], + spend=[1.0, 1.0], + ) + assert math.isfinite(prior.roi_sd) and prior.roi_sd > 0 + + def test_empty_and_length_mismatch(self): + with pytest.raises(ValueError, match="empty sequence"): + to_meridian_roi_prior(incremental_outcome=[], incremental_outcome_se=1.0, spend=1.0) + with pytest.raises(ValueError, match="length 3"): + to_meridian_roi_prior( + incremental_outcome=[1.0, 2.0], + incremental_outcome_se=[1.0, 1.0, 1.0], + spend=1.0, + ) + + +class TestToCode: + def _prior(self, parameter="roi_m"): + return to_meridian_roi_prior( + incremental_outcome=9_600.0, + incremental_outcome_se=2_400.0, + spend=20_000.0, + parameter=parameter, + ) + + def test_requires_channel_scope(self): + with pytest.raises(ValueError, match="broadcasts to"): + self._prior().to_code(full_model_window=True) + + def test_requires_time_scope(self): + with pytest.raises(ValueError, match="time scope"): + self._prior().to_code(single_channel=True) + + def test_single_channel_snippet(self): + prior = self._prior() + code = prior.to_code(single_channel=True, full_model_window=True) + assert "SINGLE-CHANNEL MODEL ONLY" in code + assert 'name="roi_m"' in code + assert 'media_prior_type="roi"' in code + assert repr(prior.mu) in code + + def test_multi_channel_vector_order_and_defaults(self): + prior = self._prior() + code = prior.to_code( + channel="tv", media_channels=["search", "tv", "social"], full_model_window=True + ) + assert repr([0.2, prior.mu, 0.2]) in code + assert repr([0.9, prior.sigma, 0.9]) in code + + def test_mroi_uses_its_default_and_prior_type(self): + prior = self._prior(parameter="mroi_m") + code = prior.to_code(channel="tv", media_channels=["search", "tv"], full_model_window=True) + assert 'name="mroi_m"' in code + assert 'media_prior_type="mroi"' in code + assert repr([0.0, prior.mu]) in code + assert repr([0.5, prior.sigma]) in code + + def test_calibration_mask_emitted(self): + code = self._prior().to_code(single_channel=True, roi_calibration_period="my_mask") + assert "roi_calibration_period=my_mask" in code + + def test_blank_or_unparseable_calibration_mask_raises(self): + for bad in ("", " ", "mask ="): + with pytest.raises(ValueError, match="valid Python expression"): + self._prior().to_code(single_channel=True, roi_calibration_period=bad) + + def test_scope_mutually_exclusive(self): + with pytest.raises(ValueError, match="not both"): + self._prior().to_code( + single_channel=True, roi_calibration_period="m", full_model_window=True + ) + with pytest.raises(ValueError, match="not both"): + self._prior().to_code( + channel="tv", + media_channels=["tv"], + single_channel=True, + full_model_window=True, + ) + + +class TestRealisticWorkflow: + """The documented use: fit an estimator, read att/se off it, scope, and export. + The module never introspects the result - the caller passes the numbers.""" + + def test_did_workflow(self): + data = generate_did_data( + n_units=60, n_periods=2, treatment_effect=5.0, treatment_period=1, seed=7 + ) + result = DifferenceInDifferences().fit( + data, outcome="outcome", treatment="treated", time="post" + ) + df = to_pymc_marketing_lift_test( + channel="tv", + x=1_000.0, + delta_x=500.0, + delta_y=result.att, + sigma=result.se, + on_wrong_sign="keep", + ) + assert df.loc[0, "delta_y"] == pytest.approx(result.att) + total, total_se = result.att * 30, result.se * 30 + prior = to_meridian_roi_prior( + incremental_outcome=total, incremental_outcome_se=total_se, spend=100_000.0 + ) + assert prior.roi_mean == pytest.approx(total / 100_000.0) + + def test_callaway_santanna_workflow(self): + # CS gives a headline ATT; the caller is responsible for turning it into a + # total incremental outcome using the estimator's own aggregation (not a + # naive att x count, which need not reproduce CS's cohort weights or + # variance - that estimator-owned aggregation is the post-4.0 follow-up). + # Here we simply confirm the fitted numbers flow through the exporter once + # the caller has supplied a total and its SE. + data = generate_staggered_data(n_units=80, n_periods=8, seed=7) + cs = CallawaySantAnna().fit( + data, outcome="outcome", unit="unit", time="period", first_treat="first_treat" + ) + assert cs.att is not None and cs.se > 0 # fit produced usable inference + # A caller-derived total incremental outcome + SE (illustrative values). + prior = to_meridian_roi_prior( + incremental_outcome=150_000.0, + incremental_outcome_se=40_000.0, + spend=200_000.0, + ) + assert prior.roi_mean == pytest.approx(150_000.0 / 200_000.0) + + +def test_public_exports(): + import diff_diff + + assert diff_diff.to_pymc_marketing_lift_test is to_pymc_marketing_lift_test + assert diff_diff.to_meridian_roi_prior is to_meridian_roi_prior + assert diff_diff.MeridianROIPrior is MeridianROIPrior