majiayu000 Circuit Breaker Pattern
Prevent cascade failures by failing fast when a service is unhealthy.
When to Use This Skill
- Adding resilience to external API calls
- Protecting against slow or failing downstream services
- Implementing graceful degradation
- Building fault-tolerant microservices
Core Concepts
A circuit breaker has three states:
- CLOSED: Normal operation, requests pass through
- OPEN: Service is failing, requests fail immediately
- HALF_OPEN: Testing if service recovered
TypeScript Implementation
// circuit-breaker.ts
type CircuitState = 'CLOSED' | 'OPEN' | 'HALF_OPEN';
interface CircuitBreakerConfig {
failureThreshold: number; // Failures before opening
successThreshold: number; // Successes to close from half-open
timeout: number; // Ms before trying half-open
fallback?: () => Promise<any>; // Optional fallback
}
class CircuitBreaker {
private state: CircuitState = 'CLOSED';
private failures = 0;
private successes = 0;
private lastFailureTime = 0;
private readonly config: CircuitBreakerConfig;
constructor(config: Partial<CircuitBreakerConfig> = {}) {
this.config = {
failureThreshold: 5,
successThreshold: 2,
timeout: 30000,
...config,
};
}
async execute<T>(fn: () => Promise<T>): Promise<T> {
if (this.state === 'OPEN') {
if (Date.now() - this.lastFailureTime >= this.config.timeout) {
this.state = 'HALF_OPEN';
} else {
if (this.config.fallback) {
return this.config.fallback();
}
throw new CircuitOpenError('Circuit is OPEN');
}
}
try {
const result = await fn();
this.onSuccess();
return result;
} catch (error) {
this.onFailure();
throw error;
}
}
private onSuccess(): void {
this.failures = 0;
if (this.state === 'HALF_OPEN') {
this.successes++;
if (this.successes >= this.config.successThreshold) {
this.state = 'CLOSED';
this.successes = 0;
}
}
}
private onFailure(): void {
this.failures++;
this.lastFailureTime = Date.now();
if (this.failures >= this.config.failureThreshold) {
this.state = 'OPEN';
}
}
getState(): CircuitState {
return this.state;
}
}
class CircuitOpenError extends Error {
constructor(message: string) {
super(message);
this.name = 'CircuitOpenError';
}
}
export { CircuitBreaker, CircuitBreakerConfig, CircuitOpenError };
Python Implementation
# circuit_breaker.py
import time
from enum import Enum
from typing import Callable, TypeVar, Optional
from functools import wraps
T = TypeVar('T')
class CircuitState(Enum):
CLOSED = "closed"
OPEN = "open"
HALF_OPEN = "half_open"
class CircuitOpenError(Exception):
pass
class CircuitBreaker:
def __init__(
self,
failure_threshold: int = 5,
success_threshold: int = 2,
timeout: float = 30.0,
fallback: Optional[Callable] = None,
):
self.failure_threshold = failure_threshold
self.success_threshold = success_threshold
self.timeout = timeout
self.fallback = fallback
self._state = CircuitState.CLOSED
self._failures = 0
self._successes = 0
self._last_failure_time = 0.0
@property
def state(self) -> CircuitState:
return self._state
def __call__(self, fn: Callable[..., T]) -> Callable[..., T]:
@wraps(fn)
def wrapper(*args, **kwargs) -> T:
return self.execute(lambda: fn(*args, **kwargs))
return wrapper
def execute(self, fn: Callable[[], T]) -> T:
if self._state == CircuitState.OPEN:
if time.time() - self._last_failure_time >= self.timeout:
self._state = CircuitState.HALF_OPEN
else:
if self.fallback:
return self.fallback()
raise CircuitOpenError("Circuit is OPEN")
try:
result = fn()
self._on_success()
return result
except Exception as e:
self._on_failure()
raise
def _on_success(self) -> None:
self._failures = 0
if self._state == CircuitState.HALF_OPEN:
self._successes += 1
if self._successes >= self.success_threshold:
self._state = CircuitState.CLOSED
self._successes = 0
def _on_failure(self) -> None:
self._failures += 1
self._last_failure_time = time.time()
if self._failures >= self.failure_threshold:
self._state = CircuitState.OPEN
Usage Examples
TypeScript with API Client
const paymentCircuit = new CircuitBreaker({
failureThreshold: 3,
timeout: 60000,
fallback: async () => ({ status: 'pending', message: 'Payment service unavailable' }),
});
async function processPayment(amount: number) {
return paymentCircuit.execute(async () => {
const response = await fetch('https://api.stripe.com/v1/charges', {
method: 'POST',
body: JSON.stringify({ amount }),
});
if (!response.ok) throw new Error('Payment failed');
return response.json();
});
}
Python Decorator Style
circuit = CircuitBreaker(failure_threshold=3, timeout=60)
@circuit
def call_external_api(endpoint: str) -> dict:
response = requests.get(endpoint, timeout=5)
response.raise_for_status()
return response.json()
Integration with Observability
Add metrics to track circuit state:
// With Prometheus
import { Gauge, Counter } from 'prom-client';
const circuitState = new Gauge({
name: 'circuit_breaker_state',
help: 'Current circuit breaker state (0=closed, 1=open, 2=half-open)',
labelNames: ['service'],
});
const circuitTrips = new Counter({
name: 'circuit_breaker_trips_total',
help: 'Total circuit breaker trips',
labelNames: ['service'],
});
Best Practices
- Tune thresholds per service: Critical services need lower thresholds
- Always provide fallbacks: Graceful degradation > hard failures
- Monitor circuit state: Alert when circuits open frequently
- Use separate circuits: One per external dependency
- Consider bulkheads: Combine with connection pooling
Common Mistakes
- Setting timeout too short (service never recovers)
- Sharing circuits across unrelated services
- Not handling CircuitOpenError in calling code
- Forgetting to add observability
Related Patterns
- Retry with exponential backoff
- Bulkhead isolation
- Timeout pattern
- Fallback pattern