levy-n Sequence Models - RNN, LSTM & Time Series
מודלים לסדרות: RNN, LSTM, חיזוי סדרות זמן, ויצירת טקסט.
Quick Start - LSTM for Time Series
import torch
import torch.nn as nn
class LSTMPredictor(nn.Module):
def __init__(self, input_size, hidden_size, num_layers=1, output_size=1):
super().__init__()
self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)
self.fc = nn.Linear(hidden_size, output_size)
def forward(self, x):
# x shape: [batch, seq_len, features]
lstm_out, (h_n, c_n) = self.lstm(x)
# Use last hidden state
out = self.fc(h_n[-1]) # [batch, output_size]
return out
# Usage
model = LSTMPredictor(input_size=1, hidden_size=64, num_layers=2)
X = torch.randn(32, 100, 1) # [batch, seq_len, features]
output = model(X) # [32, 1]
When This Skill Activates
Use this skill when:
- Working with sequential data (time series, text)
- Predicting future values in a series
- Building text generation models
- Classifying sequences (sentiment, activity)
- Understanding RNN/LSTM/GRU architectures
Core Patterns
Pattern 1: RNN vs LSTM vs GRU
| Model | Strengths | Weaknesses |
|---|---|---|
| RNN | Simple, fast | Vanishing gradients, short memory |
| LSTM | Long-term memory, gating | More parameters, slower |
| GRU | Simpler than LSTM, good memory | Less expressive than LSTM |
# RNN
rnn = nn.RNN(input_size=10, hidden_size=64, num_layers=2, batch_first=True)
# LSTM (recommended for most cases)
lstm = nn.LSTM(input_size=10, hidden_size=64, num_layers=2, batch_first=True)
# GRU (simpler alternative to LSTM)
gru = nn.GRU(input_size=10, hidden_size=64, num_layers=2, batch_first=True)
Pattern 2: LSTM Shapes (CRITICAL!)
# Input: [batch, seq_len, input_size]
# With batch_first=True (recommended)
lstm = nn.LSTM(
input_size=10, # Features per timestep
hidden_size=64, # Hidden state size
num_layers=2, # Stacked LSTM layers
batch_first=True, # [batch, seq, features] format
dropout=0.2 # Between LSTM layers (if num_layers > 1)
)
# Forward pass
X = torch.randn(32, 100, 10) # [batch, seq_len, input_size]
output, (h_n, c_n) = lstm(X)
# Outputs:
# output: [32, 100, 64] - All hidden states
# h_n: [2, 32, 64] - Final hidden state (per layer)
# c_n: [2, 32, 64] - Final cell state (per layer)
Pattern 3: Time Series Sliding Windows
def create_sequences(data, seq_length, forecast_horizon=1):
"""Create sliding window sequences for time series."""
X, y = [], []
for i in range(len(data) - seq_length - forecast_horizon + 1):
X.append(data[i:i+seq_length])
y.append(data[i+seq_length:i+seq_length+forecast_horizon])
return np.array(X), np.array(y)
# Usage
data = df['value'].values
seq_length = 30 # Use 30 timesteps to predict
forecast_horizon = 7 # Predict next 7 timesteps
X, y = create_sequences(data, seq_length, forecast_horizon)
# X shape: [num_samples, 30, 1]
# y shape: [num_samples, 7]
Pattern 4: Time Series with Multiple Features
class MultiFeatureLSTM(nn.Module):
def __init__(self, num_features, hidden_size, num_layers, output_size):
super().__init__()
self.lstm = nn.LSTM(num_features, hidden_size, num_layers,
batch_first=True, dropout=0.2)
self.fc = nn.Linear(hidden_size, output_size)
def forward(self, x):
# x: [batch, seq_len, num_features]
lstm_out, _ = self.lstm(x)
# Use last timestep output
out = self.fc(lstm_out[:, -1, :])
return out
# Example with multiple features
# Features: [temperature, humidity, pressure, hour_sin, hour_cos, is_weekend]
model = MultiFeatureLSTM(num_features=6, hidden_size=128, num_layers=2, output_size=1)
Pattern 5: Text Classification with LSTM
class TextClassifier(nn.Module):
def __init__(self, vocab_size, embed_dim, hidden_size, num_classes):
super().__init__()
self.embedding = nn.Embedding(vocab_size, embed_dim, padding_idx=0)
self.lstm = nn.LSTM(embed_dim, hidden_size, batch_first=True,
bidirectional=True)
self.fc = nn.Linear(hidden_size * 2, num_classes) # *2 for bidirectional
def forward(self, x):
# x: [batch, seq_len] - token indices
embedded = self.embedding(x) # [batch, seq_len, embed_dim]
lstm_out, (h_n, _) = self.lstm(embedded)
# Concatenate forward and backward final hidden states
hidden = torch.cat((h_n[-2], h_n[-1]), dim=1) # [batch, hidden*2]
out = self.fc(hidden)
return out
Pattern 6: Language Model (Text Generation)
class LanguageModel(nn.Module):
def __init__(self, vocab_size, embed_dim, hidden_size, num_layers):
super().__init__()
self.embedding = nn.Embedding(vocab_size, embed_dim)
self.lstm = nn.LSTM(embed_dim, hidden_size, num_layers, batch_first=True)
self.fc = nn.Linear(hidden_size, vocab_size)
def forward(self, x, hidden=None):
embedded = self.embedding(x)
output, hidden = self.lstm(embedded, hidden)
logits = self.fc(output)
return logits, hidden
def generate_text(model, seed_text, tokenizer, max_length=100, temperature=1.0):
"""Generate text autoregressively."""
model.eval()
tokens = tokenizer.encode(seed_text)
input_ids = torch.tensor([tokens])
hidden = None
generated = tokens.copy()
with torch.no_grad():
for _ in range(max_length):
logits, hidden = model(input_ids, hidden)
logits = logits[:, -1, :] / temperature
# Sample from distribution
probs = torch.softmax(logits, dim=-1)
next_token = torch.multinomial(probs, num_samples=1).item()
generated.append(next_token)
input_ids = torch.tensor([[next_token]])
if next_token == tokenizer.eos_token_id:
break
return tokenizer.decode(generated)
Pattern 7: Time Series Train/Val/Test Split
# ⚠️ CRITICAL: No shuffle for time series!
# Must maintain temporal order
def time_series_split(data, train_ratio=0.7, val_ratio=0.15):
"""Split time series maintaining order with optional gap."""
n = len(data)
train_end = int(n * train_ratio)
val_end = int(n * (train_ratio + val_ratio))
train = data[:train_end]
val = data[train_end:val_end]
test = data[val_end:]
return train, val, test
# With gap to prevent leakage
def time_series_split_with_gap(data, train_ratio=0.7, val_ratio=0.15, gap=7):
"""Add gap between splits to prevent data leakage."""
n = len(data)
train_end = int(n * train_ratio)
val_start = train_end + gap
val_end = int(n * (train_ratio + val_ratio))
test_start = val_end + gap
train = data[:train_end]
val = data[val_start:val_end]
test = data[test_start:]
return train, val, test
Pattern 8: Stacked LSTM with Dropout
class StackedLSTM(nn.Module):
def __init__(self, input_size, hidden_size, num_layers, output_size, dropout=0.2):
super().__init__()
self.lstm = nn.LSTM(
input_size=input_size,
hidden_size=hidden_size,
num_layers=num_layers,
batch_first=True,
dropout=dropout # Applied between LSTM layers
)
self.dropout = nn.Dropout(dropout) # Before final FC
self.fc = nn.Linear(hidden_size, output_size)
def forward(self, x):
lstm_out, _ = self.lstm(x)
out = self.dropout(lstm_out[:, -1, :]) # Last timestep
out = self.fc(out)
return out
Reference Navigation
For detailed content, see:
- RNN Fundamentals:
reference/rnn_fundamentals.md- RNN formula, Hidden state, Vanishing gradients - LSTM/GRU:
reference/lstm_gru.md- Architecture, Gates, Time series patterns - Text Generation:
reference/text_generation.md- Language models, Temperature sampling - Sequence Classification:
reference/sequence_classification.md- Text classification with LSTM
Common Mistakes to Avoid
1. Shuffling Time Series Data
# WRONG: DataLoader shuffles by default
train_loader = DataLoader(dataset, batch_size=32, shuffle=True) # Bad!
# CORRECT: No shuffle for time series
train_loader = DataLoader(dataset, batch_size=32, shuffle=False)
2. Fitting Scaler on All Data
# WRONG: Data leakage
scaler = StandardScaler()
data_scaled = scaler.fit_transform(all_data) # Sees future!
# CORRECT: Fit on train only
scaler = StandardScaler()
train_scaled = scaler.fit_transform(train_data)
val_scaled = scaler.transform(val_data)
test_scaled = scaler.transform(test_data)
3. Wrong LSTM Output Shape
# LSTM output: [batch, seq_len, hidden_size]
# h_n: [num_layers, batch, hidden_size]
# WRONG: Using wrong dimension
output, (h_n, c_n) = lstm(x)
final_hidden = h_n[0] # First layer, not last!
# CORRECT: Use last layer
final_hidden = h_n[-1] # Last layer's hidden state
4. Forgetting batch_first=True
# WRONG: Default is batch_first=False
lstm = nn.LSTM(10, 64)
X = torch.randn(32, 100, 10) # [batch, seq, features]
output, _ = lstm(X) # Shape mismatch!
# CORRECT: Set batch_first=True
lstm = nn.LSTM(10, 64, batch_first=True)
5. Not Detaching Hidden State
# For very long sequences, detach to prevent memory issues
# WRONG: Memory grows indefinitely
hidden = None
for batch in data_loader:
output, hidden = model(batch, hidden) # Keeps full history!
# CORRECT: Detach between batches
hidden = None
for batch in data_loader:
if hidden is not None:
hidden = (hidden[0].detach(), hidden[1].detach())
output, hidden = model(batch, hidden)
Teaching Mode
When explaining sequence models:
RNN Unrolled
Time: t-2 t-1 t t+1
│ │ │ │
▼ ▼ ▼ ▼
Input: x_{t-2} x_{t-1} x_t x_{t+1}
│ │ │ │
▼ ▼ ▼ ▼
┌───┐ ┌───┐ ┌───┐ ┌───┐
│ h │───►│ h │───►│ h │───►│ h │
└───┘ └───┘ └───┘ └───┘
│ │ │ │
▼ ▼ ▼ ▼
Output: y_{t-2} y_{t-1} y_t y_{t+1}
Hidden state h carries information forward through time
LSTM Gates Analogy
LSTM = "Memory with bouncers at the door"
Forget Gate: "Should I throw out old memories?"
Input Gate: "Should I let new info in?"
Output Gate: "What should I tell the next layer?"
The cell state is the "long-term memory"
The hidden state is the "working memory"
Vanishing Gradients
Problem with vanilla RNN:
Gradient: 1.0 → 0.5 → 0.25 → 0.125 → 0.0625 → ~0
Layer 1 2 3 4 5
Each layer multiplies gradient by <1
After many layers, gradient vanishes!
Network can't learn long-term dependencies.
LSTM solution: "Highway" for gradients through cell state