majiayu000 Model Interpretability
SHAP (SHapley Additive exPlanations)
TreeSHAP for Tree Models
import shap
def explain_tree_model(model, X_train, X_explain):
explainer = shap.TreeExplainer(model)
shap_values = explainer.shap_values(X_explain)
return explainer, shap_values
def visualize_shap(explainer, shap_values, X_explain, feature_names=None):
shap.summary_plot(shap_values, X_explain, feature_names=feature_names)
shap.dependence_plot(0, shap_values, X_explain, feature_names=feature_names)
shap.force_plot(explainer.expected_value, shap_values[0], X_explain[0],
feature_names=feature_names, matplotlib=True)
shap.waterfall_plot(shap.Explanation(
values=shap_values[0], base_values=explainer.expected_value,
data=X_explain[0], feature_names=feature_names,
))
DeepSHAP for Neural Networks
def explain_neural_network(model, X_background, X_explain):
background = torch.tensor(X_background[:100], dtype=torch.float32)
explainer = shap.DeepExplainer(model, background)
shap_values = explainer.shap_values(torch.tensor(X_explain, dtype=torch.float32))
return explainer, shap_values
def explain_image_model(model, images, class_idx=None):
explainer = shap.GradientExplainer(model, images[:50])
shap_values = explainer.shap_values(images)
shap.image_plot(shap_values, images)
return shap_values
LIME
Tabular Data
from lime import lime_tabular
def explain_with_lime(model, X_train, X_explain, feature_names, class_names=None):
explainer = lime_tabular.LimeTabularExplainer(
X_train, feature_names=feature_names, class_names=class_names,
discretize_continuous=True, random_state=42,
)
explanation = explainer.explain_instance(
X_explain[0], model.predict_proba, num_features=10, num_samples=5000,
)
for feature, weight in explanation.as_list():
print(f" {feature}: {weight:.4f}")
return explanation
Text Data
from lime.lime_text import LimeTextExplainer
def explain_text_prediction(model, text, class_names):
explainer = LimeTextExplainer(class_names=class_names)
explanation = explainer.explain_instance(
text, model.predict_proba, num_features=10, num_samples=1000,
)
return explanation
Image Data
from lime import lime_image
from skimage.segmentation import mark_boundaries
def explain_image_prediction(model, image, class_idx):
explainer = lime_image.LimeImageExplainer()
explanation = explainer.explain_instance(
image, model.predict, top_labels=5, hide_color=0, num_samples=1000,
)
temp, mask = explanation.get_image_and_mask(
class_idx, positive_only=True, num_features=5, hide_rest=False,
)
plt.imshow(mark_boundaries(temp / 255.0, mask))
return explanation
Attention Visualization
def visualize_attention(attention_weights, tokens, layer=0, head=0):
attn = attention_weights[layer, head].detach().cpu().numpy()
sns.heatmap(attn, xticklabels=tokens, yticklabels=tokens, cmap='viridis')
return attn
def attention_rollout(attentions, add_residual=True):
"""Compute attention rollout for global token importance.
Attentions: (layers, batch, heads, seq, seq)"""
attn = attentions.mean(dim=2)
if add_residual:
eye = torch.eye(attn.size(-1), device=attn.device)
attn = 0.5 * attn + 0.5 * eye
rollout = attn[0]
for layer_attn in attn[1:]:
rollout = torch.matmul(layer_attn, rollout)
return rollout / rollout.sum(dim=-1, keepdim=True)
Attention Caveats
Attention weights are NOT explanations!
- Attention != importance - tokens can be important without high attention
- Raw attention ignores value vectors and downstream processing
- Gradient-based methods are often more faithful
Use for: understanding information flow, debugging, qualitative analysis. Do NOT use for: feature importance claims, regulatory explanations.
def gradient_attention_attribution(model, input_ids, target_idx):
"""More faithful attention attribution using gradients."""
outputs = model(input_ids, output_attentions=True)
target_logit = outputs.logits[0, target_idx]
attention_grads = torch.autograd.grad(target_logit, outputs.attentions, retain_graph=True)
return [attn * grad for attn, grad in zip(outputs.attentions, attention_grads)]
Integrated Gradients
def integrated_gradients(model, input_tensor, target_class, baseline=None, steps=50):
if baseline is None:
baseline = torch.zeros_like(input_tensor)
alphas = torch.linspace(0, 1, steps, device=input_tensor.device)
interpolated = baseline + alphas.view(-1, *[1]*(input_tensor.dim()-1)) * (input_tensor - baseline)
interpolated.requires_grad_(True)
outputs = model(interpolated)
gradients = torch.autograd.grad(outputs=outputs[:, target_class].sum(), inputs=interpolated)[0]
avg_gradients = gradients.mean(dim=0)
return (input_tensor - baseline) * avg_gradients
Probing Classifiers
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import cross_val_score
class ProbingClassifier:
def __init__(self, model, layer_name):
self.model = model
self.activations = []
# Register forward hook on target layer
for name, module in model.named_modules():
if name == layer_name:
module.register_forward_hook(lambda m, i, o: self.activations.append(o.detach().cpu()))
def probe(self, representations, labels):
X = representations.view(representations.size(0), -1).numpy()
y = labels.numpy()
probe = LogisticRegression(max_iter=1000, random_state=42)
scores = cross_val_score(probe, X, y, cv=5, scoring='accuracy')
return {"mean_accuracy": scores.mean(), "std_accuracy": scores.std()}
Captum Library
from captum.attr import IntegratedGradients, GradientShap, DeepLift, Saliency
def comprehensive_attribution(model, input_tensor, target):
results = {}
results["integrated_gradients"] = IntegratedGradients(model).attribute(input_tensor, target=target)
baseline = torch.zeros_like(input_tensor)
results["gradient_shap"] = GradientShap(model).attribute(input_tensor, baselines=baseline, target=target)
results["deeplift"] = DeepLift(model).attribute(input_tensor, target=target)
results["saliency"] = Saliency(model).attribute(input_tensor, target=target)
return results
Key Pitfalls
- Trusting a single method: Different methods highlight different aspects
- Ignoring baseline choice: SHAP/IG results depend heavily on baseline
- Attention as explanation: Attention weights are not importances
- Post-hoc rationalization: Explanations may not reflect true reasoning