Experiment Tracking

Experiment Tracking Patterns

Weights & Biases

import wandb

wandb.init(
    project="fraud-detection",
    name="lgbm-v3",
    config={
        "learning_rate": 0.05,
        "n_estimators": 500,
        "max_depth": 6,
        "num_leaves": 31,
    },
    tags=["lgbm", "tabular"],
)

for epoch, (train_auc, val_auc) in enumerate(training_loop()):
    wandb.log({"train/auc": train_auc, "val/auc": val_auc, "epoch": epoch})

wandb.log({"val/confusion_matrix": wandb.plot.confusion_matrix(
    probs=None, y_true=y_true, preds=y_pred, class_names=["neg", "pos"]
)})

artifact = wandb.Artifact("model", type="model")
artifact.add_file("model.pkl")
wandb.log_artifact(artifact)
wandb.finish()

W&B Sweeps

sweep_config = {
    "method": "bayes",
    "metric": {"name": "val/auc", "goal": "maximize"},
    "parameters": {
        "learning_rate": {"distribution": "log_uniform_values", "min": 1e-4, "max": 1e-1},
        "n_estimators": {"values": [200, 500, 1000]},
        "max_depth": {"min": 3, "max": 10},
    },
    "early_terminate": {"type": "hyperband", "min_iter": 5},
}

def train_sweep():
    with wandb.init() as run:
        cfg = run.config
        model = LGBMClassifier(**cfg)
        model.fit(X_train, y_train, eval_set=[(X_val, y_val)])
        auc = roc_auc_score(y_val, model.predict_proba(X_val)[:, 1])
        wandb.log({"val/auc": auc})

sweep_id = wandb.sweep(sweep_config, project="fraud-detection")
wandb.agent(sweep_id, function=train_sweep, count=50)

MLflow

import mlflow
import mlflow.sklearn

mlflow.set_tracking_uri("http://mlflow-server:5000")
mlflow.set_experiment("fraud-detection")

with mlflow.start_run(run_name="lgbm-baseline"):
    mlflow.log_params({"lr": 0.05, "n_estimators": 500})
    mlflow.log_metrics({"train_auc": 0.92, "val_auc": 0.89})
    mlflow.sklearn.log_model(model, artifact_path="model",
                             registered_model_name="FraudClassifier")
    mlflow.log_artifact("feature_importance.png")

# Load from registry
model = mlflow.sklearn.load_model("models:/FraudClassifier/Production")

MLflow Autologging

import mlflow.lightgbm

mlflow.lightgbm.autolog(log_input_examples=True, log_model_signatures=True)

with mlflow.start_run():
    model = LGBMClassifier(n_estimators=500)
    model.fit(X_train, y_train, eval_set=[(X_val, y_val)])
    # Automatically logs params, metrics, and model artifact

Comparing Runs

# MLflow programmatic comparison
from mlflow.tracking import MlflowClient

client = MlflowClient()
runs = client.search_runs(
    experiment_ids=["1"],
    filter_string="metrics.val_auc > 0.88",
    order_by=["metrics.val_auc DESC"],
    max_results=10,
)
for run in runs:
    print(run.info.run_id, run.data.metrics["val_auc"], run.data.params)

Best Practices

• Log all hyperparameters, even defaults — reproducibility requires it
• Tag runs with git_commit, dataset_version, feature_set
• Use mlflow.set_tag("status", "production") to mark promoted models
• Store artifacts (plots, confusion matrices, SHAP) alongside metrics
• Never rely on wall-clock time to identify experiments — use run IDs