Feature Engineering

Feature Engineering Patterns

Feature Selection

from sklearn.feature_selection import (
    SelectKBest, mutual_info_classif, RFE, SelectFromModel
)
from sklearn.ensemble import RandomForestClassifier
import shap

# Mutual information
selector = SelectKBest(score_func=mutual_info_classif, k=20)
X_selected = selector.fit_transform(X_train, y_train)
selected_features = X_train.columns[selector.get_support()].tolist()

# Recursive Feature Elimination
rfe = RFE(estimator=RandomForestClassifier(n_estimators=100), n_features_to_select=15)
rfe.fit(X_train, y_train)
rfe_features = X_train.columns[rfe.support_].tolist()

# SHAP-based selection
model = RandomForestClassifier().fit(X_train, y_train)
explainer = shap.TreeExplainer(model)
shap_values = explainer.shap_values(X_train)
mean_abs_shap = pd.DataFrame({
    "feature": X_train.columns,
    "importance": np.abs(shap_values[1]).mean(axis=0),
}).sort_values("importance", ascending=False)

Encoding Patterns

import pandas as pd
import numpy as np
from sklearn.preprocessing import OrdinalEncoder, TargetEncoder
from category_encoders import WOEEncoder

# Target encoding (safe with cross-val)
te = TargetEncoder(smooth="auto")
X_train["city_encoded"] = te.fit_transform(X_train[["city"]], y_train)
X_test["city_encoded"] = te.transform(X_test[["city"]])

# Ordinal encoding with unknown handling
oe = OrdinalEncoder(handle_unknown="use_encoded_value", unknown_value=-1)
X_train[["size"]] = oe.fit_transform(X_train[["size"]])

# Weight of Evidence (for binary classification)
woe = WOEEncoder(cols=["category"])
X_train_woe = woe.fit_transform(X_train, y_train)
X_test_woe = woe.transform(X_test)

# Frequency encoding
freq_map = X_train["product"].value_counts(normalize=True).to_dict()
X_train["product_freq"] = X_train["product"].map(freq_map)
X_test["product_freq"] = X_test["product"].map(freq_map).fillna(0)

Scaling

from sklearn.preprocessing import StandardScaler, RobustScaler, QuantileTransformer
from sklearn.pipeline import Pipeline
from sklearn.compose import ColumnTransformer

numeric_features = ["amount", "age", "duration"]
categorical_features = ["city", "product"]

preprocessor = ColumnTransformer([
    ("num", RobustScaler(), numeric_features),       # robust to outliers
    ("cat", TargetEncoder(), categorical_features),
])

pipeline = Pipeline([
    ("preprocessor", preprocessor),
    ("model", RandomForestClassifier()),
])
pipeline.fit(X_train, y_train)

Time-Series Features

def add_time_features(df: pd.DataFrame, ts_col: str) -> pd.DataFrame:
    df = df.copy()
    ts = pd.to_datetime(df[ts_col])
    df["hour"] = ts.dt.hour
    df["dow"] = ts.dt.dayofweek
    df["month"] = ts.dt.month
    df["is_weekend"] = (ts.dt.dayofweek >= 5).astype(int)
    df["quarter"] = ts.dt.quarter
    # Cyclical encoding
    df["hour_sin"] = np.sin(2 * np.pi * df["hour"] / 24)
    df["hour_cos"] = np.cos(2 * np.pi * df["hour"] / 24)
    df["dow_sin"] = np.sin(2 * np.pi * df["dow"] / 7)
    df["dow_cos"] = np.cos(2 * np.pi * df["dow"] / 7)
    return df

def add_lag_features(df: pd.DataFrame, col: str, lags: list[int]) -> pd.DataFrame:
    df = df.sort_values("ts")
    for lag in lags:
        df[f"{col}_lag_{lag}"] = df.groupby("user_id")[col].shift(lag)
    df[f"{col}_rolling_7d"] = (
        df.groupby("user_id")[col]
        .transform(lambda x: x.rolling(7, min_periods=1).mean())
    )
    return df

Feast Feature Store

from feast import FeatureStore, Entity, FeatureView, Field, FileSource
from feast.types import Float64, Int64

store = FeatureStore(repo_path="feature_repo/")

# Define entity
user = Entity(name="user_id", join_keys=["user_id"])

# Define feature view
user_features = FeatureView(
    name="user_stats",
    entities=[user],
    ttl=timedelta(days=7),
    schema=[
        Field(name="daily_spend", dtype=Float64),
        Field(name="tx_count", dtype=Int64),
    ],
    source=FileSource(path="data/user_stats.parquet", timestamp_field="event_timestamp"),
)

# Retrieve features for training
entity_df = pd.DataFrame({"user_id": user_ids, "event_timestamp": timestamps})
training_df = store.get_historical_features(
    entity_df=entity_df,
    features=["user_stats:daily_spend", "user_stats:tx_count"],
).to_df()

# Online retrieval
    features=["user_stats:daily_spend", "user_stats:tx_count"],
    entity_rows=[{"user_id": "u123"}],
).to_dict()

Anti-Patterns

• Target leakage: never use post-event data as features; always validate with time-based splits
• Train-test contamination: fit encoders on train only, transform both
• High cardinality raw: raw user IDs as features cause memorization — encode or embed
• Scaling tree models: tree-based models don't need feature scaling; only linear/SVM/NN do