Eve library

SLO Policies

Guide

Revision 1

On this page

Comprehensive guide for Service Level Objectives (SLOs) and error budget management in the Lilith platform.

Table of Contents#

Overview#

SLOs define the target reliability level for services and help balance reliability work against feature development.

Key Concepts#

Term Definition
SLI Service Level Indicator - metric measuring service behavior
SLO Service Level Objective - target value for an SLI
SLA Service Level Agreement - contractual commitment
Error Budget Allowed unreliability (100% - SLO)

SLO Hierarchy#

text
┌─────────────────────────────────────────────────────────────────────────┐
│                          SLO Architecture                                │
│                                                                          │
│  ┌─────────────────────────────────────────────────────────────────┐    │
│  │                     Platform SLOs                                │    │
│  │   Availability: 99.9%    Latency P99: 500ms    Error: < 0.1%   │    │
│  └─────────────────────────────────────────────────────────────────┘    │
│                                │                                         │
│         ┌──────────────────────┼──────────────────────┐                 │
│         ▼                      ▼                      ▼                 │
│  ┌─────────────┐       ┌─────────────┐       ┌─────────────┐           │
│  │   BFF SLO   │       │  Auth SLO   │       │Content SLO  │           │
│  │ Avail: 99.9%│       │ Avail: 99.95│       │ Avail: 99.9%│           │
│  │ P99: 200ms  │       │ P99: 100ms  │       │ P99: 300ms  │           │
│  └─────────────┘       └─────────────┘       └─────────────┘           │
│                                                                          │
└─────────────────────────────────────────────────────────────────────────┘

SLO Framework#

SLI Types#

Availability SLI#

promql
# Request success rate
sum(rate(http_requests_total{status!~"5.."}[5m])) by (service)
/
sum(rate(http_requests_total[5m])) by (service)

Latency SLI#

promql
# P99 latency
histogram_quantile(0.99,
  sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)
)

# Good latency ratio (requests under threshold)
sum(rate(http_request_duration_seconds_bucket{le="0.5"}[5m])) by (service)
/
sum(rate(http_request_duration_seconds_count[5m])) by (service)

Throughput SLI#

promql
# Requests per second
sum(rate(http_requests_total[5m])) by (service)

Correctness SLI#

promql
# Data consistency checks passed
sum(rate(data_consistency_checks_passed_total[5m]))
/
sum(rate(data_consistency_checks_total[5m]))

SLO Windows#

Window Use Case Calculation
Rolling 30-day Primary SLO compliance Continuous
Rolling 7-day Short-term trends Continuous
Calendar month Business reporting Monthly reset
promql
# 30-day availability
avg_over_time(
  (
    sum(rate(http_requests_total{status!~"5.."}[5m])) by (service)
    /
    sum(rate(http_requests_total[5m])) by (service)
  )[30d:5m]
)

Service SLOs#

Platform-Wide SLOs#

Service Availability Latency P99 Error Rate
BFF 99.9% < 200ms < 0.1%
Auth 99.95% < 100ms < 0.05%
Content 99.9% < 300ms < 0.1%
AI 99.5% < 5000ms < 0.5%
Media 99.9% < 500ms < 0.1%
Notification 99.5% < 1000ms < 0.5%

Service-Specific SLO Definitions#

Auth Service#

yaml
service: auth
slos:
  - name: availability
    description: 'Auth service request success rate'
    target: 99.95%
    window: 30d
    sli:
      type: availability
      query: |
        sum(rate(http_requests_total{service="auth",status!~"5.."}[5m]))
        /
        sum(rate(http_requests_total{service="auth"}[5m]))

  - name: latency-p99
    description: 'Auth service P99 latency'
    target: 100ms
    window: 30d
    sli:
      type: latency
      query: |
        histogram_quantile(0.99,
          sum(rate(http_request_duration_seconds_bucket{service="auth"}[5m])) by (le)
        )

  - name: token-validation-latency
    description: 'Token validation P99 latency'
    target: 50ms
    window: 30d
    sli:
      type: latency
      query: |
        histogram_quantile(0.99,
          sum(rate(auth_token_validation_duration_seconds_bucket[5m])) by (le)
        )

BFF Service#

yaml
service: bff
slos:
  - name: availability
    target: 99.9%
    window: 30d
    sli:
      query: |
        sum(rate(http_requests_total{service="bff",status!~"5.."}[5m]))
        /
        sum(rate(http_requests_total{service="bff"}[5m]))

  - name: latency-p99
    target: 200ms
    window: 30d
    sli:
      query: |
        histogram_quantile(0.99,
          sum(rate(http_request_duration_seconds_bucket{service="bff"}[5m])) by (le)
        )

  - name: graphql-errors
    description: 'GraphQL query error rate'
    target: 0.1%
    window: 30d
    sli:
      query: |
        sum(rate(graphql_requests_total{status="error"}[5m]))
        /
        sum(rate(graphql_requests_total[5m]))

AI Service#

yaml
service: ai
slos:
  - name: availability
    target: 99.5%
    window: 30d
    sli:
      query: |
        sum(rate(http_requests_total{service="ai",status!~"5.."}[5m]))
        /
        sum(rate(http_requests_total{service="ai"}[5m]))

  - name: inference-latency-p99
    target: 5000ms
    window: 30d
    sli:
      query: |
        histogram_quantile(0.99,
          sum(rate(ai_inference_duration_seconds_bucket[5m])) by (le)
        )

  - name: model-load-success
    description: 'Model loading success rate'
    target: 99.9%
    window: 30d
    sli:
      query: |
        sum(rate(ai_model_load_total{status="success"}[5m]))
        /
        sum(rate(ai_model_load_total[5m]))

Critical User Journeys#

Login Journey SLO#

yaml
journey: user-login
description: 'Complete user login flow'
slos:
  - name: success-rate
    target: 99.9%
    sli:
      query: |
        sum(rate(user_login_total{status="success"}[5m]))
        /
        sum(rate(user_login_total[5m]))

  - name: latency-p99
    target: 2000ms
    sli:
      query: |
        histogram_quantile(0.99,
          sum(rate(user_login_duration_seconds_bucket[5m])) by (le)
        )

Content Playback SLO#

yaml
journey: content-playback
description: 'Start playing meditation or lecture'
slos:
  - name: time-to-first-byte
    target: 1000ms
    sli:
      query: |
        histogram_quantile(0.99,
          sum(rate(content_playback_start_duration_seconds_bucket[5m])) by (le)
        )

  - name: buffering-ratio
    target: 0.5%
    sli:
      query: |
        sum(rate(content_buffering_seconds_total[5m]))
        /
        sum(rate(content_playback_seconds_total[5m]))

Error Budgets#

Error Budget Calculation#

text
Error Budget = 1 - SLO Target

For 99.9% availability SLO:
Error Budget = 1 - 0.999 = 0.001 = 0.1%

Monthly budget (30 days):
0.001 × 30 × 24 × 60 = 43.2 minutes of downtime

Error Budget Table#

SLO Target Error Budget Monthly Downtime
99.99% 0.01% 4.32 minutes
99.95% 0.05% 21.6 minutes
99.9% 0.1% 43.2 minutes
99.5% 0.5% 3.6 hours
99% 1% 7.2 hours

Error Budget Consumption#

promql
# Error budget remaining (percentage)
1 - (
  (1 - avg_over_time(service:availability:ratio[30d]))
  /
  (1 - 0.999)  # SLO target
)

Error Budget Policy#

yaml
error_budget_policy:
  # Normal operation
  - remaining: '> 50%'
    actions:
      - continue_feature_development: true
      - reliability_work_percentage: 20%

  # Warning zone
  - remaining: '25-50%'
    actions:
      - reduce_risk_deployments: true
      - reliability_work_percentage: 40%
      - review_recent_incidents: true

  # Critical zone
  - remaining: '< 25%'
    actions:
      - freeze_feature_releases: true
      - reliability_work_percentage: 80%
      - incident_review_required: true
      - executive_notification: true

  # Exhausted
  - remaining: '0%'
    actions:
      - production_freeze: true
      - reliability_work_percentage: 100%
      - mandatory_postmortem: true
      - leadership_escalation: true

SLO Monitoring#

Prometheus Recording Rules#

yaml
# deploy/terraform/modules/observability/prometheus/rules/slo.yaml
groups:
  - name: slo-recording
    interval: 1m
    rules:
      # Availability SLI
      - record: service:availability:ratio
        expr: |
          sum(rate(http_requests_total{status!~"5.."}[5m])) by (service)
          /
          sum(rate(http_requests_total[5m])) by (service)

      # Latency P99
      - record: service:latency:p99
        expr: |
          histogram_quantile(0.99,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service)
          )

      # Latency SLI (requests under threshold)
      - record: service:latency:ratio
        expr: |
          sum(rate(http_request_duration_seconds_bucket{le="0.5"}[5m])) by (service)
          /
          sum(rate(http_request_duration_seconds_count[5m])) by (service)

      # Error rate
      - record: service:error:ratio
        expr: |
          sum(rate(http_requests_total{status=~"5.."}[5m])) by (service)
          /
          sum(rate(http_requests_total[5m])) by (service)

      # 30-day averages
      - record: service:availability:ratio30d
        expr: avg_over_time(service:availability:ratio[30d])

      - record: service:latency:p99_30d
        expr: avg_over_time(service:latency:p99[30d])

Grafana Dashboard#

json
{
  "dashboard": {
    "title": "SLO Dashboard",
    "panels": [
      {
        "title": "Availability SLO",
        "type": "gauge",
        "targets": [
          {
            "expr": "service:availability:ratio30d{service=\"auth\"} * 100",
            "legendFormat": "Auth"
          }
        ],
        "fieldConfig": {
          "defaults": {
            "thresholds": {
              "steps": [
                { "value": 0, "color": "red" },
                { "value": 99, "color": "yellow" },
                { "value": 99.9, "color": "green" }
              ]
            },
            "min": 95,
            "max": 100,
            "unit": "percent"
          }
        }
      },
      {
        "title": "Error Budget Remaining",
        "type": "stat",
        "targets": [
          {
            "expr": "(1 - ((1 - service:availability:ratio30d) / (1 - 0.999))) * 100"
          }
        ]
      },
      {
        "title": "Error Budget Burn Rate",
        "type": "timeseries",
        "targets": [
          {
            "expr": "(1 - service:availability:ratio) / (1 - 0.999)",
            "legendFormat": "{{ service }}"
          }
        ]
      }
    ]
  }
}

SLO Alerting#

Multi-Window Multi-Burn-Rate Alerts#

yaml
groups:
  - name: slo-alerts
    rules:
      # Fast burn - 2% of error budget in 1 hour
      - alert: SLOHighBurnRate
        expr: |
          (
            (1 - service:availability:ratio1h) / (1 - 0.999)
          ) > 14.4
          and
          (
            (1 - service:availability:ratio5m) / (1 - 0.999)
          ) > 14.4
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: 'High SLO burn rate for {{ $labels.service }}'
          description:
            'Burning error budget at {{ $value | printf "%.1f" }}x normal rate'

      # Slow burn - 10% of error budget in 6 hours
      - alert: SLOSlowBurnRate
        expr: |
          (
            (1 - service:availability:ratio6h) / (1 - 0.999)
          ) > 6
          and
          (
            (1 - service:availability:ratio30m) / (1 - 0.999)
          ) > 6
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: 'Elevated SLO burn rate for {{ $labels.service }}'

      # Error budget exhausted
      - alert: SLOErrorBudgetExhausted
        expr: |
          (1 - service:availability:ratio30d) > (1 - 0.999)
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: 'Error budget exhausted for {{ $labels.service }}'

Latency SLO Alerts#

yaml
rules:
  - alert: LatencySLOBreach
    expr: |
      service:latency:p99 > 0.5
      and
      service:latency:ratio < 0.999
    for: 5m
    labels:
      severity: warning
    annotations:
      summary: 'Latency SLO breach for {{ $labels.service }}'
      description:
        'P99 latency is {{ $value | printf "%.2f" }}s (target: 500ms)'

Burn Rate#

Burn Rate Concept#

text
Burn Rate = Actual Error Rate / Allowed Error Rate

For 99.9% SLO (0.1% error budget):
- Normal operation: burn rate = 1.0 (using budget at expected pace)
- 2x burn rate: exhausts monthly budget in 15 days
- 10x burn rate: exhausts monthly budget in 3 days
- 36x burn rate: exhausts monthly budget in 20 hours

Burn Rate Windows#

Alert Burn Rate Short Window Long Window Budget Consumed
Page 14.4x 5m 1h 2%
Page 6x 30m 6h 5%
Ticket 3x 2h 1d 10%
Ticket 1x 6h 3d 10%

Recording Rules for Burn Rate#

yaml
groups:
  - name: slo-burn-rate
    rules:
      # 5-minute window
      - record: service:availability:ratio5m
        expr: |
          sum(rate(http_requests_total{status!~"5.."}[5m])) by (service)
          /
          sum(rate(http_requests_total[5m])) by (service)

      # 30-minute window
      - record: service:availability:ratio30m
        expr: |
          sum(rate(http_requests_total{status!~"5.."}[30m])) by (service)
          /
          sum(rate(http_requests_total[30m])) by (service)

      # 1-hour window
      - record: service:availability:ratio1h
        expr: |
          sum(rate(http_requests_total{status!~"5.."}[1h])) by (service)
          /
          sum(rate(http_requests_total[1h])) by (service)

      # 6-hour window
      - record: service:availability:ratio6h
        expr: |
          sum(rate(http_requests_total{status!~"5.."}[6h])) by (service)
          /
          sum(rate(http_requests_total[6h])) by (service)

      # Burn rate calculation
      - record: service:slo:burn_rate
        labels:
          slo: availability
          target: '0.999'
        expr: |
          (1 - service:availability:ratio5m)
          /
          (1 - 0.999)

SLO Review Process#

Monthly SLO Review#

markdown
## SLO Review Agenda

### 1. SLO Performance Summary

- [ ] Review 30-day SLO achievement for each service
- [ ] Identify services that missed SLO
- [ ] Analyze error budget consumption trends

### 2. Incident Impact Analysis

- [ ] Review incidents that impacted SLOs
- [ ] Calculate total SLO impact per incident
- [ ] Identify recurring issues

### 3. SLO Adjustments

- [ ] Propose SLO target changes based on:
  - User expectations
  - Business requirements
  - Technical capabilities
- [ ] Document rationale for changes

### 4. Action Items

- [ ] Reliability improvements needed
- [ ] Monitoring gaps to address
- [ ] SLO refinements to implement

SLO Change Process#

  1. Propose Change: Document reason for SLO change
  2. Impact Assessment: Calculate new error budget
  3. Stakeholder Review: Engineering, Product, Leadership
  4. Implementation: Update monitoring, alerting, dashboards
  5. Communication: Announce changes to team

Implementation#

Sloth Configuration#

yaml
# deploy/terraform/modules/observability/slo/sloth.yaml
version: 'prometheus/v1'
service: 'lilith'
labels:
  team: platform
slos:
  - name: 'auth-availability'
    objective: 99.95
    description: 'Auth service availability'
    sli:
      events:
        error_query: sum(rate(http_requests_total{service="auth",status=~"5.."}[{{.window}}]))
        total_query: sum(rate(http_requests_total{service="auth"}[{{.window}}]))
    alerting:
      name: AuthAvailability
      labels:
        team: auth
      annotations:
        runbook: 'https://docs.lilith.app/runbooks/auth-availability'
      page_alert:
        labels:
          severity: critical
      ticket_alert:
        labels:
          severity: warning

  - name: 'bff-latency'
    objective: 99.9
    description: 'BFF P99 latency under 200ms'
    sli:
      events:
        error_query: |
          sum(rate(http_request_duration_seconds_count{service="bff"}[{{.window}}]))
          -
          sum(rate(http_request_duration_seconds_bucket{service="bff",le="0.2"}[{{.window}}]))
        total_query: sum(rate(http_request_duration_seconds_count{service="bff"}[{{.window}}]))
    alerting:
      name: BFFLatency
      annotations:
        runbook: 'https://docs.lilith.app/runbooks/bff-latency'

Terraform Module#

hcl
# deploy/terraform/modules/observability/slo/main.tf

resource "kubernetes_config_map" "slo_rules" {
  metadata {
    name      = "slo-recording-rules"
    namespace = "monitoring"
    labels = {
      "prometheus" = "main"
    }
  }

  data = {
    "slo-recording.yaml" = file("${path.module}/rules/slo-recording.yaml")
    "slo-alerting.yaml"  = file("${path.module}/rules/slo-alerting.yaml")
  }
}

resource "grafana_dashboard" "slo" {
  folder      = grafana_folder.slo.id
  config_json = file("${path.module}/dashboards/slo.json")
}

Service Integration#

typescript
// packages/metrics/src/slo.ts

export interface SLOConfig {
  service: string;
  availability: {
    target: number; // e.g., 0.999
    window: '7d' | '30d';
  };
  latency: {
    p99Target: number; // milliseconds
    window: '7d' | '30d';
  };
}

export function registerSLOMetrics(config: SLOConfig) {
  // Request counter for availability SLI
  const requestCounter = new Counter({
    name: 'http_requests_total',
    help: 'Total HTTP requests',
    labelNames: ['service', 'method', 'status', 'path'],
  });

  // Histogram for latency SLI
  const latencyHistogram = new Histogram({
    name: 'http_request_duration_seconds',
    help: 'HTTP request duration',
    labelNames: ['service', 'method', 'path'],
    buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 1, 2, 5],
  });

  return { requestCounter, latencyHistogram };
}