# Monitoring Setup and Procedures

**Purpose**: Comprehensive monitoring, alerting, and observability configuration
for Lilith platform

## Monitoring Stack Overview

### Architecture

```
Applications → Metrics Collection → Storage → Visualization → Alerting
     ↓               ↓                ↓           ↓           ↓
Services         Prometheus        TSDB       Grafana    AlertManager
   +             +Exporters         +           +           +
Logs            Fluentd/Promtail   Loki       Grafana    PagerDuty
   +                ↓                ↓           ↓           +
Traces          OpenTelemetry      Tempo      Grafana     Slack
```

### Core Components

```
Metrics: Prometheus + Grafana
Logs: Loki + Grafana
Traces: Tempo + Grafana
Alerting: AlertManager + PagerDuty
Dashboards: Grafana + Custom panels
```

## Prometheus Configuration

### Service Discovery and Scraping

#### Core Services Monitoring

```yaml
# prometheus-config.yaml
global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
  # Kubernetes API server
  - job_name: 'kubernetes-apiservers'
    kubernetes_sd_configs:
      - role: endpoints
    scheme: https
    tls_config:
      ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
    bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
    relabel_configs:
      - source_labels:
          [
            __meta_kubernetes_namespace,
            __meta_kubernetes_service_name,
            __meta_kubernetes_endpoint_port_name,
          ]
        action: keep
        regex: default;kubernetes;https

  # Application services
  - job_name: 'lilith-services'
    kubernetes_sd_configs:
      - role: pod
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels:
          [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: kubernetes_pod_name

  # Database monitoring
  - job_name: 'postgres-exporter'
    static_configs:
      - targets: ['postgres-exporter:9187']
    scrape_interval: 30s

  # Infrastructure monitoring
  - job_name: 'node-exporter'
    kubernetes_sd_configs:
      - role: node
    relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
```

### Service-Specific Metrics

#### BFF Service Metrics

```yaml
# BFF service annotations for Prometheus scraping
apiVersion: apps/v1
kind: Deployment
metadata:
  name: bff-deployment
  namespace: lilith-bff
spec:
  template:
    metadata:
      annotations:
        prometheus.io/scrape: 'true'
        prometheus.io/port: '8080'
        prometheus.io/path: '/metrics'
    spec:
      containers:
        - name: bff
          image: lilith/bff:latest
          ports:
            - containerPort: 8080
              name: http
            - containerPort: 9090
              name: metrics
```

#### Custom Application Metrics

```javascript
// Example: BFF service metrics implementation
const client = require('prom-client');

// Create a Registry
const register = new client.Registry();

// HTTP request metrics
const httpRequestDuration = new client.Histogram({
  name: 'http_request_duration_seconds',
  help: 'Duration of HTTP requests in seconds',
  labelNames: ['method', 'route', 'status_code'],
  buckets: [0.1, 0.3, 0.5, 0.7, 1, 3, 5, 7, 10],
});

const httpRequestsTotal = new client.Counter({
  name: 'http_requests_total',
  help: 'Total number of HTTP requests',
  labelNames: ['method', 'route', 'status_code'],
});

// Business metrics
const activeUsers = new client.Gauge({
  name: 'active_users_total',
  help: 'Number of currently active users',
});

const conversationsCreated = new client.Counter({
  name: 'conversations_created_total',
  help: 'Total number of conversations created',
});

// Register metrics
register.registerMetric(httpRequestDuration);
register.registerMetric(httpRequestsTotal);
register.registerMetric(activeUsers);
register.registerMetric(conversationsCreated);

// Metrics endpoint
app.get('/metrics', async (req, res) => {
  res.set('Content-Type', register.contentType);
  res.end(await register.metrics());
});
```

## Grafana Dashboard Configuration

### Service-Level Dashboards

#### BFF Service Dashboard

```json
{
  "dashboard": {
    "title": "BFF Service Overview",
    "panels": [
      {
        "title": "Request Rate",
        "type": "stat",
        "targets": [
          {
            "expr": "sum(rate(http_requests_total{job=\"bff\"}[5m]))",
            "legendFormat": "Requests/sec"
          }
        ],
        "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }
      },
      {
        "title": "Error Rate",
        "type": "stat",
        "targets": [
          {
            "expr": "sum(rate(http_requests_total{job=\"bff\",status=~\"5..\"}[5m])) / sum(rate(http_requests_total{job=\"bff\"}[5m])) * 100",
            "legendFormat": "Error %"
          }
        ],
        "thresholds": [
          { "color": "green", "value": null },
          { "color": "yellow", "value": 2 },
          { "color": "red", "value": 5 }
        ],
        "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 }
      },
      {
        "title": "Response Time (95th percentile)",
        "type": "stat",
        "targets": [
          {
            "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"bff\"}[5m])) by (le))",
            "legendFormat": "95th percentile"
          }
        ],
        "unit": "s",
        "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 }
      },
      {
        "title": "Active Connections",
        "type": "stat",
        "targets": [
          {
            "expr": "sum(active_users_total{job=\"bff\"})",
            "legendFormat": "Active Users"
          }
        ],
        "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 }
      }
    ],
    "time": { "from": "now-1h", "to": "now" },
    "refresh": "30s"
  }
}
```

#### Platform Overview Dashboard

```json
{
  "dashboard": {
    "title": "Lilith Platform Overview",
    "panels": [
      {
        "title": "Service Health Status",
        "type": "stat",
        "targets": [
          {
            "expr": "up{job=~\"bff|auth|conversation|content|media|ai\"}",
            "legendFormat": "{{job}}"
          }
        ],
        "valueMappings": [
          { "value": 1, "text": "UP", "color": "green" },
          { "value": 0, "text": "DOWN", "color": "red" }
        ]
      },
      {
        "title": "Request Volume by Service",
        "type": "graph",
        "targets": [
          {
            "expr": "sum by (job) (rate(http_requests_total[5m]))",
            "legendFormat": "{{job}}"
          }
        ]
      },
      {
        "title": "Error Rate by Service",
        "type": "graph",
        "targets": [
          {
            "expr": "sum by (job) (rate(http_requests_total{status=~\"5..\"}[5m])) / sum by (job) (rate(http_requests_total[5m])) * 100",
            "legendFormat": "{{job}} errors"
          }
        ]
      },
      {
        "title": "Database Connections",
        "type": "graph",
        "targets": [
          {
            "expr": "pg_stat_database_numbackends{datname=\"lilith\"}",
            "legendFormat": "Active connections"
          }
        ]
      }
    ]
  }
}
```

## Alerting Rules and Configuration

### Prometheus Alerting Rules

#### Critical Service Alerts

```yaml
# alerts/critical-services.yml
groups:
  - name: critical-services
    rules:
      # Service down alert
      - alert: ServiceDown
        expr: up{job=~"bff|auth|conversation"} == 0
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: 'Service {{$labels.job}} is down'
          description:
            'Service {{$labels.job}} has been down for more than 1 minute.'

      # High error rate alert
      - alert: HighErrorRate
        expr: |
          (
            sum(rate(http_requests_total{status=~"5.."}[5m])) by (job)
            /
            sum(rate(http_requests_total[5m])) by (job)
          ) * 100 > 5
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: 'High error rate on {{$labels.job}}'
          description: 'Error rate is {{$value}}% for {{$labels.job}}'

      # High response time alert
      - alert: HighResponseTime
        expr: |
          histogram_quantile(0.95,
            sum(rate(http_request_duration_seconds_bucket[5m])) by (le, job)
          ) > 2
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: 'High response time on {{$labels.job}}'
          description:
            '95th percentile response time is {{$value}}s for {{$labels.job}}'

      # Database connection pool exhaustion
      - alert: DatabaseConnectionPoolHigh
        expr: pg_stat_database_numbackends{datname="lilith"} > 80
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: 'Database connection pool usage high'
          description: 'Database has {{$value}} active connections'

      # Pod crash loop
      - alert: PodCrashLooping
        expr: rate(kube_pod_container_status_restarts_total[15m]) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: 'Pod {{$labels.pod}} is crash looping'
          description:
            'Pod {{$labels.pod}} in namespace {{$labels.namespace}} is
            restarting frequently'
```

### AlertManager Configuration

#### Routing and Notification Setup

```yaml
# alertmanager.yml
global:
  smtp_smarthost: 'mail.lilith.com:587'
  smtp_from: 'alerts@lilith.com'

route:
  group_by: ['alertname', 'cluster', 'service']
  group_wait: 10s
  group_interval: 10s
  repeat_interval: 12h
  receiver: 'web.hook'
  routes:
    - match:
        severity: critical
      receiver: 'pagerduty-critical'
      repeat_interval: 5m
    - match:
        severity: warning
      receiver: 'slack-warnings'
      repeat_interval: 30m

receivers:
  - name: 'web.hook'
    webhook_configs:
      - url: 'http://webhook.lilith.com/alerts'

  - name: 'pagerduty-critical'
    pagerduty_configs:
      - service_key: 'your-pagerduty-integration-key'
        description: '{{range .Alerts}}{{.Annotations.summary}}{{end}}'

  - name: 'slack-warnings'
    slack_configs:
      - api_url: 'YOUR_SLACK_WEBHOOK_URL'
        channel: '#lilith-alerts'
        title: 'Alert: {{.GroupLabels.alertname}}'
        text: '{{range .Alerts}}{{.Annotations.description}}{{end}}'

inhibit_rules:
  - source_match:
      severity: 'critical'
    target_match:
      severity: 'warning'
    equal: ['alertname', 'cluster', 'service']
```

## Log Aggregation and Analysis

### Loki Configuration for Log Aggregation

#### Loki Configuration

```yaml
# loki-config.yml
auth_enabled: false

server:
  http_listen_port: 3100

ingester:
  lifecycler:
    address: 127.0.0.1
    ring:
      kvstore:
        store: inmemory
      replication_factor: 1
    final_sleep: 0s

schema_config:
  configs:
    - from: 2020-10-24
      store: boltdb-shipper
      object_store: filesystem
      schema: v11
      index:
        prefix: index_
        period: 24h

storage_config:
  boltdb_shipper:
    active_index_directory: /loki/boltdb-shipper-active
    cache_location: /loki/boltdb-shipper-cache
    shared_store: filesystem
  filesystem:
    directory: /loki/chunks

limits_config:
  enforce_metric_name: false
  reject_old_samples: true
  reject_old_samples_max_age: 168h

chunk_store_config:
  max_look_back_period: 0s

table_manager:
  retention_deletes_enabled: false
  retention_period: 0s
```

### Structured Logging Setup

#### Application Logging Standards

```javascript
// Example: Structured logging in Node.js services
const winston = require('winston');

const logger = winston.createLogger({
  level: 'info',
  format: winston.format.combine(
    winston.format.timestamp(),
    winston.format.errors({ stack: true }),
    winston.format.json()
  ),
  defaultMeta: {
    service: process.env.SERVICE_NAME || 'unknown',
    version: process.env.SERVICE_VERSION || 'unknown',
    environment: process.env.NODE_ENV || 'development',
  },
  transports: [
    new winston.transports.Console(),
    new winston.transports.File({
      filename: '/var/log/app/error.log',
      level: 'error',
    }),
    new winston.transports.File({ filename: '/var/log/app/combined.log' }),
  ],
});

// Usage examples
logger.info('User authentication successful', {
  userId: '12345',
  email: 'user@example.com',
  duration: 150,
  endpoint: '/api/auth/login',
});

logger.error('Database connection failed', {
  error: error.message,
  stack: error.stack,
  database: 'primary',
  retryAttempt: 3,
});

logger.warn('High response time detected', {
  endpoint: '/api/chat/messages',
  responseTime: 2500,
  threshold: 2000,
});
```

## Distributed Tracing Setup

### OpenTelemetry Configuration

#### Trace Collection Setup

```javascript
// OpenTelemetry setup for Node.js services
const { NodeTracerProvider } = require('@opentelemetry/sdk-node');
const { Resource } = require('@opentelemetry/resources');
const {
  SemanticResourceAttributes,
} = require('@opentelemetry/semantic-conventions');
const { JaegerExporter } = require('@opentelemetry/exporter-jaeger');
const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-base');

// Initialize tracing
const provider = new NodeTracerProvider({
  resource: new Resource({
    [SemanticResourceAttributes.SERVICE_NAME]: process.env.SERVICE_NAME,
    [SemanticResourceAttributes.SERVICE_VERSION]: process.env.SERVICE_VERSION,
  }),
});

// Configure Jaeger exporter
const jaegerExporter = new JaegerExporter({
  endpoint: process.env.JAEGER_ENDPOINT || 'http://jaeger:14268/api/traces',
});

provider.addSpanProcessor(new BatchSpanProcessor(jaegerExporter));
provider.register();

// Auto-instrument HTTP requests
const {
  getNodeAutoInstrumentations,
} = require('@opentelemetry/auto-instrumentations-node');
const { registerInstrumentations } = require('@opentelemetry/instrumentation');

registerInstrumentations({
  instrumentations: [
    getNodeAutoInstrumentations({
      '@opentelemetry/instrumentation-fs': { enabled: false },
    }),
  ],
});
```

## Monitoring Automation Scripts

### Health Check Automation

```bash
#!/bin/bash
# Automated health check and alerting script

SERVICES=("bff" "auth" "conversation" "content" "media" "ai")
HEALTH_CHECK_LOG="/var/log/health-checks.log"

perform_health_check() {
  local service=$1
  local health_url="https://${service}.lilith.com/v1/health"

  echo "$(date): Checking $service health..." >> $HEALTH_CHECK_LOG

  response=$(curl -s -w "%{http_code}:%{time_total}" "$health_url")
  http_code=$(echo $response | cut -d: -f1)
  response_time=$(echo $response | cut -d: -f2)

  if [ "$http_code" = "200" ]; then
    echo "$(date): ✓ $service healthy (${response_time}s)" >> $HEALTH_CHECK_LOG

    # Check if response time is acceptable
    if (( $(echo "$response_time > 2.0" | bc -l) )); then
      ./scripts/send-alert.sh "⚠️ $service slow response time: ${response_time}s"
    fi
  else
    echo "$(date): ✗ $service unhealthy (HTTP $http_code)" >> $HEALTH_CHECK_LOG
    ./scripts/send-alert.sh "🚨 $service health check failed (HTTP $http_code)"
  fi
}

# Check all services
for service in "${SERVICES[@]}"; do
  perform_health_check "$service" &
done

wait
echo "$(date): Health check cycle completed" >> $HEALTH_CHECK_LOG
```

### Metrics Collection Validation

```bash
#!/bin/bash
# Validate that metrics are being collected properly

validate_metrics_collection() {
  local service=$1
  local prometheus_url="http://prometheus.lilith.com:9090"

  echo "Validating metrics collection for $service..."

  # Check if service is being scraped
  scrape_status=$(curl -s "$prometheus_url/api/v1/query?query=up{job=\"$service\"}" | jq -r '.data.result[0].value[1]')

  if [ "$scrape_status" = "1" ]; then
    echo "✓ $service metrics are being scraped"
  else
    echo "✗ $service metrics scraping failed"
    return 1
  fi

  # Check if custom metrics are present
  custom_metrics=$(curl -s "$prometheus_url/api/v1/query?query=http_requests_total{job=\"$service\"}" | jq -r '.data.result | length')

  if [ "$custom_metrics" -gt 0 ]; then
    echo "✓ $service custom metrics are available"
  else
    echo "✗ $service custom metrics missing"
    return 1
  fi

  echo "✓ Metrics validation completed for $service"
}

# Validate all services
for service in bff auth conversation content; do
  validate_metrics_collection "$service"
done
```

---

**Remember**: Effective monitoring requires the right balance of coverage, noise
reduction, and actionable alerts. Focus on metrics that indicate user impact and
business value, not just technical health.
