Model Evaluation Patterns
Classification Metrics
from sklearn.metrics import (
roc_auc_score, average_precision_score, f1_score,
classification_report, confusion_matrix, ConfusionMatrixDisplay
)
import matplotlib.pyplot as plt
y_prob = model.predict_proba(X_test)[:, 1]
y_pred = (y_prob >= 0.5).astype(int)
print(f"AUC-ROC: {roc_auc_score(y_test, y_prob):.4f}")
print(f"AUC-PR: {average_precision_score(y_test, y_prob):.4f}")
print(f"F1: {f1_score(y_test, y_pred):.4f}")
print(classification_report(y_test, y_pred, target_names=["neg", "pos"]))
ConfusionMatrixDisplay.from_predictions(y_test, y_pred, display_labels=["neg", "pos"])
plt.savefig("confusion_matrix.png", dpi=150, bbox_inches="tight")
Calibration
from sklearn.calibration import calibration_curve, CalibratedClassifierCV
import matplotlib.pyplot as plt
# Check calibration
fraction_pos, mean_pred = calibration_curve(y_test, y_prob, n_bins=10)
plt.plot(mean_pred, fraction_pos, "s-", label="Model")
plt.plot([0, 1], [0, 1], "k--", label="Perfect")
plt.xlabel("Mean predicted probability")
plt.ylabel("Fraction of positives")
plt.title("Calibration curve")
plt.legend()
# Recalibrate with Platt scaling or isotonic regression
calibrated = CalibratedClassifierCV(model, method="isotonic", cv=5)
calibrated.fit(X_train, y_train)
Bootstrap Confidence Intervals
import numpy as np
from sklearn.utils import resample
def bootstrap_metric(y_true, y_prob, metric_fn, n_iterations=1000, ci=0.95):
scores = []
rng = np.random.default_rng(42)
for _ in range(n_iterations):
idx = resample(np.arange(len(y_true)), random_state=rng)
scores.append(metric_fn(y_true[idx], y_prob[idx]))
lower = np.percentile(scores, (1 - ci) / 2 * 100)
upper = np.percentile(scores, (1 + ci) / 2 * 100)
return np.mean(scores), lower, upper
mean, lo, hi = bootstrap_metric(y_test, y_prob, roc_auc_score)
print(f"AUC = {mean:.4f} (95% CI: {lo:.4f}–{hi:.4f})")
SHAP Explanations
import shap
explainer = shap.TreeExplainer(model)
shap_values = explainer.shap_values(X_test)
# Global importance
shap.summary_plot(shap_values, X_test, plot_type="bar", max_display=15)
# Single prediction explanation
shap.waterfall_plot(shap.Explanation(
values=shap_values[0],
base_values=explainer.expected_value,
data=X_test.iloc[0],
feature_names=X_test.columns.tolist(),
))
Fairness Metrics
from sklearn.metrics import confusion_matrix
def fairness_report(y_true, y_pred, sensitive_attr):
groups = {}
for grp in sensitive_attr.unique():
mask = sensitive_attr == grp
tn, fp, fn, tp = confusion_matrix(y_true[mask], y_pred[mask]).ravel()
groups[grp] = {
"tpr": tp / (tp + fn), # recall / sensitivity
"fpr": fp / (fp + tn), # false positive rate
"ppv": tp / (tp + fp), # precision
"n": mask.sum(),
}
return pd.DataFrame(groups).T
report = fairness_report(y_test, y_pred, df_test["gender"])
print(report)
# Check: |TPR_A - TPR_B| < 0.05 (equal opportunity)
Regression Metrics
from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score
import numpy as np
rmse = np.sqrt(mean_squared_error(y_test, y_pred))
mae = mean_absolute_error(y_test, y_pred)
r2 = r2_score(y_test, y_pred)
mape = np.mean(np.abs((y_test - y_pred) / np.clip(np.abs(y_test), 1e-8, None))) * 100
print(f"RMSE={rmse:.2f} MAE={mae:.2f} R²={r2:.4f} MAPE={mape:.2f}%")
Key Patterns
- For imbalanced data: use AUC-PR over AUC-ROC; threshold tune on F1 or business metric
- Always report metrics with confidence intervals, not just point estimates
- Stratified k-fold ensures class balance across folds for classification
- Report per-class metrics, not just macro average — minority class often matters most
Mattakushi432