Conditional average treatment effects with PLR¶
This notebook estimates heterogeneous treatment effects using the Best Linear Predictor (BLP) approach (Semenova & Chernozhukov, 2021) on top of a Partially Linear Regression (PLR) model.
The PLR model is: $$Y = D\theta_0 + g_0(X) + \zeta, \qquad \mathbb{E}(\zeta | D, X) = 0,$$ $$D = m_0(X) + V, \qquad \mathbb{E}(V | X) = 0.$$
While PLR yields a single Average Treatment Effect (ATE) $\theta_0$, the BLP framework lets us recover conditional treatment effects $\tau(x) = \mathbb{E}[\theta(X) \mid B(X) = b]$ by projecting the score onto flexible basis functions $B(X)$ (e.g. B-splines).
Workflow
- Spark session + data loading
- Nuisance fitting (keeping
df_nuisance) - ATE estimation + sensitivity
- 1-D CATEs along individual covariates (B-spline BLP)
- 2-D CATE surface over two covariates (tensor-product B-splines)
- Cleanup
1. Imports and Configuration¶
import os
from glob import glob
import numpy as np
import pandas as pd
import patsy
from scipy.stats import norm
from matplotlib import pyplot as plt
from doubleml_pyspark.plm.plr import DoubleMLPLR
from doubleml_pyspark.utils import kfold_split
from pyspark.ml.regression import LinearRegression, RandomForestRegressor
from pyspark.sql import SparkSession
from pyspark.sql.functions import monotonically_increasing_id, col, round as spark_round
from pyspark.sql.types import DoubleType, FloatType, DecimalType
plt.rcParams['figure.figsize'] = (12, 7)
plt.rcParams['font.size'] = 12
plt.rcParams['axes.grid'] = True
plt.rcParams['grid.alpha'] = 0.3
# ── Configuration ──────────────────────────────────────────────────────────
DATA_PATH = os.path.abspath("../data/plr_data/")
N_FEATURES = 5
N_FOLDS = 3
SAMPLE_FRACTION = None # set e.g. 0.1 for 10 % during development
print(f"Data path : {DATA_PATH}")
print(f"Features : {N_FEATURES}")
print(f"CV folds : {N_FOLDS}")
print(f"Sample fraction: {SAMPLE_FRACTION or 'full dataset'}")
Data path : /workspace/sandbox/data/plr_data Features : 5 CV folds : 3 Sample fraction: full dataset
2. Spark Session¶
A local session with G1GC, Kryo serialization, and AQE enabled.
Increase driver_memory if you hit OOM on large datasets.
def initialize_spark_session(
app_name="DoubleMLPLR_CATE",
driver_memory="4g",
executor_memory="4g",
max_result_size="2g",
):
"""Build a SparkSession tuned for DoubleML PLR workloads."""
spark = (
SparkSession.builder
.appName(app_name)
.config("spark.driver.memory", driver_memory)
.config("spark.executor.memory", executor_memory)
.config("spark.driver.maxResultSize", max_result_size)
.config("spark.sql.shuffle.partitions", "200")
.config("spark.sql.files.maxPartitionBytes", "32m")
.config("spark.default.parallelism", "200")
.config("spark.sql.adaptive.enabled", "true")
.config("spark.sql.adaptive.coalescePartitions.enabled", "true")
.config("spark.sql.autoBroadcastJoinThreshold", "50m")
.config("spark.serializer",
"org.apache.spark.serializer.KryoSerializer")
.config("spark.driver.extraJavaOptions", "-XX:+UseG1GC")
.config("spark.executor.extraJavaOptions", "-XX:+UseG1GC")
.getOrCreate()
)
return spark
spark = initialize_spark_session()
print("Spark session created successfully.")
WARNING: Using incubator modules: jdk.incubator.vector Using Spark's default log4j profile: org/apache/spark/log4j2-defaults.properties Setting default log level to "WARN". To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel). 26/04/27 12:32:09 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 26/04/27 12:32:11 WARN Utils: Service 'SparkUI' could not bind on port 4040. Attempting port 4041.
Spark session created successfully.
3. Data Loading and Preparation¶
def load_data(data_path, sample_fraction=None):
"""Load parquet files; optionally subsample for development."""
files = glob(os.path.join(data_path, "*.parquet"))
if not files:
# fallback to IRM data directory
alt = data_path.replace("plr_data", "irm_data")
files = glob(os.path.join(alt, "chunk_*.parquet"))
if not files:
raise FileNotFoundError(f"No parquet files found in {data_path}")
print(f"Found {len(files)} file(s). Loading...")
df = spark.read.parquet(*files)
if sample_fraction and sample_fraction < 1.0:
print(f"Sampling {sample_fraction * 100:.0f}% of data (seed=42)...")
df = df.sample(fraction=sample_fraction, seed=42)
n = df.count()
print(f"Rows: {n:,} | Columns: {df.columns}")
return df
def prepare_data(df):
"""
Attach a unique ID, round floats to 4 dp for numerical stability,
and cache the DataFrame.
Note: `monotonically_increasing_id()` is globally unique but non-contiguous
and not stable across re-executions — use only for join keys within a session.
"""
df = df.withColumn("ID", monotonically_increasing_id())
for field in df.schema.fields:
if isinstance(field.dataType, (DoubleType, FloatType, DecimalType)) \
and field.name != "ID":
df = df.withColumn(field.name, spark_round(col(field.name), 4))
df = df.cache()
print(f"DataFrame cached ({df.count():,} rows).")
return df
df_raw = load_data(DATA_PATH, SAMPLE_FRACTION)
df_with_id = prepare_data(df_raw)
df_with_id.show(5)
Found 10 file(s). Loading...
Rows: 10,000,000 | Columns: ['X1', 'X2', 'X3', 'X4', 'X5', 'y', 'd']
DataFrame cached (10,000,000 rows). +-------+-------+-------+-------+-------+-------+-------+-----------+ | X1| X2| X3| X4| X5| y| d| ID| +-------+-------+-------+-------+-------+-------+-------+-----------+ |-0.5305|-0.0293|-0.7574|-0.8054| 0.2615|-0.9593|-0.6535|51539607552| | 1.5042| 0.5167| 0.0462|-0.5098|-0.5239| 3.71| 2.5428|51539607553| | 0.3699| -0.002|-0.1616| 1.4997| 0.1174| 5.1316| 1.5267|51539607554| |-0.0316| 0.0718|-0.0187| 1.4367| 0.7678| 1.9758| 0.4895|51539607555| |-0.8809|-1.6079|-1.4581|-0.5893|-1.2095|-2.5507|-0.4727|51539607556| +-------+-------+-------+-------+-------+-------+-------+-----------+ only showing top 5 rows
4. Model Configuration¶
PLR requires two nuisance models. The first, ml_l, targets E[Y | X] and serves as the outcome regression. The second, ml_m, targets E[D | X] and serves as the treatment regression.
covariate_cols = [f"X{i+1}" for i in range(N_FEATURES)]
data_field_map = {
"id_column" : "ID",
"outcome" : "y",
"treatment" : "d",
"covariates": covariate_cols,
}
ml_models = {
"ml_l": LinearRegression(maxIter=100, regParam=0.01), # E[Y|X]
"ml_m": LinearRegression(maxIter=100, regParam=0.01), # E[D|X]
# Uncomment for non-linear nuisance:
# "ml_l": RandomForestRegressor(numTrees=100, maxDepth=5),
# "ml_m": RandomForestRegressor(numTrees=100, maxDepth=5),
}
print("Data field map:", data_field_map)
print("ML models :", {k: type(v).__name__ for k, v in ml_models.items()})
Data field map: {'id_column': 'ID', 'outcome': 'y', 'treatment': 'd', 'covariates': ['X1', 'X2', 'X3', 'X4', 'X5']}
ML models : {'ml_l': 'LinearRegression', 'ml_m': 'LinearRegression'}
5. Cross-Fitting: Nuisance Models + ATE¶
We call fit_ml_models() and estimate_causal_parameters() separately so that
df_nuisance stays cached for the downstream cate() calls.
(fit() would unpersist it automatically at the end.)
kf_split = kfold_split(
df_with_id, n_folds=N_FOLDS, n_rep=1, shuffle=True, seed=42
)
dml_plr = DoubleMLPLR(
data_schema=df_with_id.schema,
data_field_map=data_field_map,
)
df_nuisance = None
estimation_successful = False
try:
# ── Step 1: fit nuisance models across folds ───────────────────────────
print("Fitting nuisance models...")
df_nuisance, additional_nuisance = dml_plr.fit_ml_models(
spark_session=spark,
kfold_split=kf_split[0],
ml_models=ml_models,
)
df_nuisance = df_nuisance.cache()
print(f"df_nuisance cached ({df_nuisance.count():,} rows).")
# ── Step 2: solve the partialling-out estimating equation for θ ────────
print("Estimating causal parameters (ATE)...")
dml_plr.estimate_causal_parameters(
spark_session=spark,
df_nuisance=df_nuisance,
additional_nuisance_elements=additional_nuisance,
)
estimation_successful = True
print("\n✓ Model fitting completed successfully!")
except Exception as e:
print(f"\n✗ Error: {e}")
if "Connection refused" in str(e) or "Py4JNetworkError" in str(e):
print("Tip: set SAMPLE_FRACTION=0.1 or increase Spark driver memory.")
Fitting nuisance models...
df_nuisance cached (10,000,000 rows). Estimating causal parameters (ATE)...
[Stage 357:==================================================> (190 + 8) / 200]
✓ Model fitting completed successfully!
ATE Results¶
if estimation_successful:
theta_ate = dml_plr.thetas[0]
ci = dml_plr.confint()
print("=" * 55)
print("ATE RESULTS (PLR)")
print("=" * 55)
print(f" θ̂ (ATE) : {theta_ate:.4f}")
if hasattr(dml_plr, 'ses'):
print(f" SE : {dml_plr.ses[0]:.4f}")
print(f" 95% CI : [{ci.iloc[0, 0]:.4f}, {ci.iloc[0, 1]:.4f}]")
display(ci)
else:
print("⚠ Skipping results — estimation failed.")
======================================================= ATE RESULTS (PLR) ======================================================= θ̂ (ATE) : 1.9998 SE : 0.0003 95% CI : [1.9992, 2.0004]
| 2.5 % | 97.5 % | |
|---|---|---|
| d | 1.999166 | 2.000407 |
6. Conditional Average Treatment Effects (CATEs)¶
We use the Best Linear Predictor (BLP) approach from Semenova & Chernozhukov (2021). The CATE along covariate Xj is recovered by regressing the score on a B-spline basis B(Xj), giving a predicted treatment effect as the inner product of the basis evaluated at xj and the estimated BLP coefficients.
6.1 Confidence Intervals via DoubleMLBLP.confint()¶
DoubleMLBLP.confint(spark_session, basis, level) returns pointwise confidence intervals for the predicted CATE on a Spark DataFrame evaluation grid, using the coefficient covariance precomputed by fit(). We call it directly on each grid below — no local helper, no manual covariance reconstruction. The returned Spark DataFrame has columns for the lower bound, the effect, and the upper bound, which we convert with .toPandas() for plotting with matplotlib or Plotly.
The grid is pushed through spark.createDataFrame(...).coalesce(1) so that .toPandas() returns rows in the same order as the input grid, allowing us to align the result with the original x_vals array for plotting.
6.2 Collect a Covariate Sample to Pandas¶
Pulling the full dataset to the driver OOMs the Spark JVM on large data. We instead sample a fixed number of rows within Spark first, then collect. 100 k rows is more than enough for a consistent BLP estimate.
# Sample within Spark before collecting — avoids pulling millions of rows to the driver
N_SAMPLE_BLP = 100_000
n_total = df_with_id.count()
sample_frac = min(1.0, N_SAMPLE_BLP / n_total)
print(f"Dataset has {n_total:,} rows — sampling {sample_frac * 100:.1f}% ({N_SAMPLE_BLP:,} rows) for BLP basis.")
df_cov_pd = (
df_with_id
.select("ID", *covariate_cols)
.sample(fraction=sample_frac, seed=42)
.toPandas()
)
print(f"Collected {len(df_cov_pd):,} rows.")
df_cov_pd.head(3)
Dataset has 10,000,000 rows — sampling 1.0% (100,000 rows) for BLP basis. Collected 99,603 rows.
| ID | X1 | X2 | X3 | X4 | X5 | |
|---|---|---|---|---|---|---|
| 0 | 51539607638 | -0.4642 | 0.0378 | 1.5529 | 1.2982 | 2.2008 |
| 1 | 51539607713 | -0.1743 | 0.9582 | 0.2179 | 0.0413 | 0.0885 |
| 2 | 51539607944 | 0.3104 | 0.1803 | -0.0087 | -0.4320 | 0.2464 |
6.3 One-Dimensional CATE along X1¶
We build a degree-2 B-spline basis on X1 (6 columns), fit the BLP model,
then evaluate on a fine grid with uniform 95 % confidence bands.
# ── Build B-spline training basis ─────────────────────────────────────────
dm_x1 = patsy.dmatrix(
"bs(x, df=5, degree=2)",
{"x": df_cov_pd["X1"]},
return_type="matrix", # retains .design_info for grid evaluation
)
bcols_x1 = [f"basis_{i}" for i in range(dm_x1.shape[1])]
basis_pd_x1 = pd.DataFrame(np.asarray(dm_x1), columns=bcols_x1)
basis_pd_x1["ID"] = df_cov_pd["ID"].values
df_basis_x1 = spark.createDataFrame(basis_pd_x1)
print(f"X1 basis: {len(bcols_x1)} cols × {len(basis_pd_x1):,} rows")
# ── Fit BLP ───────────────────────────────────────────────────────────────
blp_x1 = dml_plr.cate(
spark_session=spark,
df_basis=df_basis_x1,
df_nuisance=df_nuisance,
)
print("BLP fitted for X1.")
X1 basis: 6 cols × 99,603 rows
26/04/27 12:41:01 WARN Instrumentation: [dbf7d9f8] regParam is zero, which might cause numerical instability and overfitting. [Stage 549:=======================================> (148 + 8) / 200]
BLP fitted for X1.
# ── Evaluate on grid via DoubleMLBLP.confint() ────────────────────────────
x1_vals = np.linspace(df_cov_pd["X1"].quantile(0.05),
df_cov_pd["X1"].quantile(0.95), 100)
grid_x1 = pd.DataFrame(
patsy.build_design_matrices([dm_x1.design_info], {"x": x1_vals})[0],
columns=bcols_x1,
)
df_cate_x1 = (
blp_x1.confint(
spark_session=spark,
basis=spark.createDataFrame(grid_x1).coalesce(1),
level=0.95,
)
.toPandas()
)
df_cate_x1["x"] = x1_vals
# ── Plot ──────────────────────────────────────────────────────────────────
fig, ax = plt.subplots(figsize=(9, 5))
ax.plot(df_cate_x1["x"], df_cate_x1["effect"],
color="#2563eb", linewidth=2, label="CATE")
ax.fill_between(df_cate_x1["x"],
df_cate_x1["2.5 %"], df_cate_x1["97.5 %"],
color="#2563eb", alpha=0.15, label="Pointwise 95% CI")
if estimation_successful:
ax.axhline(theta_ate, color="#dc2626", linestyle="--",
linewidth=1.5, alpha=0.8, label=f"ATE = {theta_ate:.3f}")
ax.set_xlabel("X1"); ax.set_ylabel("Treatment Effect")
ax.set_title("CATE along X1 (PLR · BLP)")
ax.legend()
plt.tight_layout()
plt.show()
6.4 One-Dimensional CATEs across All Covariates¶
A quick pass over every covariate to spot which ones drive treatment effect heterogeneity. Covariates whose CATE curve is flat and overlaps the ATE line are likely irrelevant moderators.
cov = "X1"
dm = patsy.dmatrix("bs(x, df=5, degree=2)", {"x": df_cov_pd[cov]})
bcols = [f"basis_{j}" for j in range(dm.shape[1])]
bpd = pd.DataFrame(np.asarray(dm), columns=bcols)
bpd["ID"] = df_cov_pd["ID"].values
blp = dml_plr.cate(
spark_session=spark,
df_basis=spark.createDataFrame(bpd),
df_nuisance=df_nuisance,
)
x_vals = np.linspace(df_cov_pd[cov].quantile(0.05),
df_cov_pd[cov].quantile(0.95), 100)
grid = pd.DataFrame(
patsy.build_design_matrices([dm.design_info], {"x": x_vals})[0],
columns=bcols,
)
df_ci = (
blp.confint(
spark_session=spark,
basis=spark.createDataFrame(grid).coalesce(1),
level=0.95,
)
.toPandas()
)
df_ci["x"] = x_vals
fig, ax = plt.subplots(figsize=(8, 5))
ax.plot(df_ci["x"], df_ci["effect"], color="#2563eb", linewidth=2, label="CATE")
ax.fill_between(df_ci["x"], df_ci["2.5 %"], df_ci["97.5 %"],
color="#2563eb", alpha=0.15, label="Pointwise 95% CI")
if estimation_successful:
ax.axhline(theta_ate, color="#dc2626", linestyle="--",
linewidth=1.5, alpha=0.8, label=f"ATE = {theta_ate:.3f}")
ax.set_xlabel(cov)
ax.set_ylabel("Treatment Effect")
ax.set_title(f"CATE along {cov} (PLR · BLP)")
ax.legend()
plt.tight_layout()
plt.show()
26/04/27 12:42:10 WARN Instrumentation: [67f175f2] regParam is zero, which might cause numerical instability and overfitting.
7. Cleanup¶
if df_nuisance is not None:
df_nuisance.unpersist()
print("✓ df_nuisance unpersisted.")
try:
spark.stop()
print("✓ Spark session stopped.")
except Exception as e:
print(f"⚠ Cleanup warning: {e}")
✓ df_nuisance unpersisted. ✓ Spark session stopped.
References¶
Chernozhukov, V., Chetverikov, D., Demirer, M., Duflo, E., Hansen, C., Newey, W., & Robins, J. (2018).
Double/debiased machine learning for treatment and structural parameters.
The Econometrics Journal, 21(1), C1–C68.Semenova, V., & Chernozhukov, V. (2021).
Debiased machine learning of conditional average treatment effects and other causal functions.
The Econometrics Journal, 24(2), 264–289.