Golang Observability

Go Observability Patterns

Structured Logging with slog (Go 1.21+)

import "log/slog"

func main() {
    // JSON handler for production
    logger := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
        Level:     slog.LevelInfo,
        AddSource: true,
    }))
    slog.SetDefault(logger)

    // Structured fields
    slog.Info("server started", "port", 8080, "env", "production")
    slog.Error("request failed", "err", err, "method", r.Method, "path", r.URL.Path)
    slog.Warn("rate limit exceeded", "ip", clientIP, "limit", rps)
}

// Logger with context (request-scoped)
func withRequestLogger(ctx context.Context, requestID string) *slog.Logger {
    return slog.Default().With("requestID", requestID)
}

// Middleware that injects logger into context
func loggingMiddleware(logger *slog.Logger) func(http.Handler) http.Handler {
    return func(next http.Handler) http.Handler {
        return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
            reqLogger := logger.With(
                "requestID", r.Header.Get("X-Request-ID"),
                "method", r.Method,
                "path", r.URL.Path,
            )
            ctx := context.WithValue(r.Context(), loggerKey{}, reqLogger)
            start := time.Now()
            next.ServeHTTP(w, r.WithContext(ctx))
            reqLogger.Info("request completed", "duration", time.Since(start))
        })
    }
}

func LoggerFromContext(ctx context.Context) *slog.Logger {
    if l, ok := ctx.Value(loggerKey{}).(*slog.Logger); ok { return l }
    return slog.Default()
}

Prometheus Metrics

import (
    "github.com/prometheus/client_golang/prometheus"
    "github.com/prometheus/client_golang/prometheus/promauto"
    "github.com/prometheus/client_golang/prometheus/promhttp"
)

var (
    httpRequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{
        Name: "http_requests_total",
        Help: "Total HTTP requests",
    }, []string{"method", "path", "status"})

    httpRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{
        Name:    "http_request_duration_seconds",
        Help:    "HTTP request duration",
        Buckets: prometheus.DefBuckets,
    }, []string{"method", "path"})

    activeConnections = promauto.NewGauge(prometheus.GaugeOpts{
        Name: "active_connections",
        Help: "Number of active connections",
    })
)

// Instrumented response writer
type statusRecorder struct {
    http.ResponseWriter
    status int
}

func (r *statusRecorder) WriteHeader(status int) {
    r.status = status
    r.ResponseWriter.WriteHeader(status)
}

func metricsMiddleware(next http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        rec := &statusRecorder{ResponseWriter: w, status: 200}
        start := time.Now()
        activeConnections.Inc()
        defer activeConnections.Dec()

        next.ServeHTTP(rec, r)

        httpRequestsTotal.WithLabelValues(r.Method, r.URL.Path, strconv.Itoa(rec.status)).Inc()
        httpRequestDuration.WithLabelValues(r.Method, r.URL.Path).Observe(time.Since(start).Seconds())
    })
}

// Expose metrics endpoint
mux.Handle("/metrics", promhttp.Handler())

OpenTelemetry Tracing

import (
    "go.opentelemetry.io/otel"
    "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
    "go.opentelemetry.io/otel/sdk/trace"
)

func initTracer(endpoint string) (func(context.Context) error, error) {
    exporter, err := otlptracegrpc.New(context.Background(),
        otlptracegrpc.WithEndpoint(endpoint),
        otlptracegrpc.WithInsecure(),
    )
    if err != nil { return nil, err }

    tp := trace.NewTracerProvider(
        trace.WithBatcher(exporter),
        trace.WithResource(resource.NewWithAttributes(
            semconv.SchemaURL,
            semconv.ServiceName("my-service"),
            semconv.ServiceVersion("1.0.0"),
        )),
    )
    otel.SetTracerProvider(tp)
    return tp.Shutdown, nil
}

// Instrument a function
var tracer = otel.Tracer("my-service")

func (s *Service) ProcessOrder(ctx context.Context, orderID string) error {
    ctx, span := tracer.Start(ctx, "ProcessOrder")
    defer span.End()

    span.SetAttributes(attribute.String("order.id", orderID))

    if err := s.repo.GetOrder(ctx, orderID); err != nil {
        span.RecordError(err)
        span.SetStatus(codes.Error, err.Error())
        return err
    }
    return nil
}

Common Anti-Patterns

• log.Printf in new code — use slog for structured, level-aware logging
• Logging inside library packages — accept a logger via interface; don't use global logger in libs
• High-cardinality label values in Prometheus — never use user IDs or paths with variables as labels
• Not sampling traces — 100% trace sampling in high-traffic services is expensive; use head-based sampling
• Missing span propagation — always pass ctx through your call stack to maintain trace context