eliferjunior PagerDuty
Overview
Set up PagerDuty for incident management with on-call schedules, escalation policies, and integrations. Covers service creation, Events API for triggering alerts, schedule management, and automation via the REST API.
Instructions
Task A: Create Services and Escalation Policies
# Create an escalation policy
curl -X POST "https://api.pagerduty.com/escalation_policies" \
-H "Authorization: Token token=${PD_API_KEY}" \
-H "Content-Type: application/json" \
-d '{
"escalation_policy": {
"name": "Platform Team Escalation",
"escalation_rules": [
{
"escalation_delay_in_minutes": 10,
"targets": [
{ "id": "P1AB2CD", "type": "schedule_reference" }
]
},
{
"escalation_delay_in_minutes": 15,
"targets": [
{ "id": "PXYZ789", "type": "user_reference" }
]
}
],
"repeat_enabled": true,
"num_loops": 2
}
}'
# Create a service with the escalation policy
curl -X POST "https://api.pagerduty.com/services" \
-H "Authorization: Token token=${PD_API_KEY}" \
-H "Content-Type: application/json" \
-d '{
"service": {
"name": "Payment Service - Production",
"description": "Payment processing microservice",
"escalation_policy": { "id": "PESCAL1", "type": "escalation_policy_reference" },
"alert_creation": "create_alerts_and_incidents",
"auto_resolve_timeout": 14400,
"acknowledgement_timeout": 1800,
"alert_grouping_parameters": {
"type": "intelligent"
},
"incident_urgency_rule": {
"type": "use_support_hours",
"during_support_hours": { "type": "constant", "urgency": "high" },
"outside_support_hours": { "type": "constant", "urgency": "low" }
},
"support_hours": {
"type": "fixed_time_per_day",
"time_zone": "America/New_York",
"days_of_week": [1, 2, 3, 4, 5],
"start_time": "08:00:00",
"end_time": "20:00:00"
}
}
}'
Task B: Send Alerts via Events API
# Trigger an alert
curl -X POST "https://events.pagerduty.com/v2/enqueue" \
-H "Content-Type: application/json" \
-d '{
"routing_key": "<INTEGRATION_KEY>",
"event_action": "trigger",
"dedup_key": "payment-service/high-error-rate/prod",
"payload": {
"summary": "Payment Service: Error rate exceeded 5% (currently 8.3%)",
"severity": "critical",
"source": "prometheus-alertmanager",
"component": "payment-service",
"group": "production",
"class": "error_rate",
"custom_details": {
"error_rate": "8.3%",
"threshold": "5%",
"affected_endpoints": ["/api/charge", "/api/refund"],
"runbook": "https://wiki.internal/runbooks/payment-errors"
}
},
"links": [
{ "href": "https://grafana.internal/d/payments", "text": "Grafana Dashboard" }
]
}'
# Acknowledge an alert
curl -X POST "https://events.pagerduty.com/v2/enqueue" \
-H "Content-Type: application/json" \
-d '{
"routing_key": "<INTEGRATION_KEY>",
"event_action": "acknowledge",
"dedup_key": "payment-service/high-error-rate/prod"
}'
# Resolve an alert
curl -X POST "https://events.pagerduty.com/v2/enqueue" \
-H "Content-Type: application/json" \
-d '{
"routing_key": "<INTEGRATION_KEY>",
"event_action": "resolve",
"dedup_key": "payment-service/high-error-rate/prod"
}'
Task C: On-Call Schedules
# Create a weekly rotation schedule
curl -X POST "https://api.pagerduty.com/schedules" \
-H "Authorization: Token token=${PD_API_KEY}" \
-H "Content-Type: application/json" \
-d '{
"schedule": {
"name": "Platform Team Primary On-Call",
"time_zone": "America/New_York",
"schedule_layers": [
{
"name": "Weekly Rotation",
"start": "2026-02-19T09:00:00-05:00",
"rotation_virtual_start": "2026-02-19T09:00:00-05:00",
"rotation_turn_length_seconds": 604800,
"users": [
{ "user": { "id": "PUSER01", "type": "user_reference" } },
{ "user": { "id": "PUSER02", "type": "user_reference" } },
{ "user": { "id": "PUSER03", "type": "user_reference" } }
],
"restrictions": [
{
"type": "daily_restriction",
"start_time_of_day": "09:00:00",
"duration_seconds": 57600
}
]
}
]
}
}'
# Get who is currently on call
curl -s "https://api.pagerduty.com/oncalls?schedule_ids[]=PSCHED1&earliest=true" \
-H "Authorization: Token token=${PD_API_KEY}" | \
jq '.oncalls[] | {user: .user.summary, schedule: .schedule.summary, start: .start, end: .end}'
Task D: Incident Management
# List open incidents
curl -s "https://api.pagerduty.com/incidents?statuses[]=triggered&statuses[]=acknowledged" \
-H "Authorization: Token token=${PD_API_KEY}" | \
jq '.incidents[] | {id: .id, title: .title, status: .status, urgency: .urgency, service: .service.summary, created: .created_at}'
# Add a note to an incident
curl -X POST "https://api.pagerduty.com/incidents/${INCIDENT_ID}/notes" \
-H "Authorization: Token token=${PD_API_KEY}" \
-H "Content-Type: application/json" \
-H "From: oncall@example.com" \
-d '{
"note": {
"content": "Identified root cause: connection pool exhaustion on db-primary. Scaling up connections from 100 to 200."
}
}'
Task E: Automation with Event Orchestration
# Create event orchestration rules
curl -X PUT "https://api.pagerduty.com/event_orchestrations/${ORCH_ID}/router" \
-H "Authorization: Token token=${PD_API_KEY}" \
-H "Content-Type: application/json" \
-d '{
"orchestration_path": {
"sets": [
{
"id": "start",
"rules": [
{
"label": "Route payments alerts",
"conditions": [
{ "expression": "event.component matches part '''payment''' " }
],
"actions": {
"route_to": { "service": { "id": "PSVC_PAY", "type": "service_reference" } }
}
},
{
"label": "Suppress health checks",
"conditions": [
{ "expression": "event.payload.summary matches part '''health check'''" }
],
"actions": { "suppress": true }
}
]
}
],
"catch_all": {
"actions": {
"route_to": { "service": { "id": "PSVC_DEFAULT", "type": "service_reference" } }
}
}
}
}'
Best Practices
- Use
dedup_keyto prevent duplicate incidents for the same issue - Set intelligent alert grouping to automatically correlate related alerts
- Include runbook links and dashboard URLs in alert custom details
- Configure support hours to route low-urgency alerts during business hours only
- Rotate on-call weekly and limit shifts to avoid burnout
- Use event orchestration to suppress noisy alerts and enrich events before routing