{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "description": "Lilith event handler pipeline: dispatch rate by event type, projection lag, and dead-letter queue depth.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "liveNow": false, "panels": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Events dispatched per second by event type.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": true, "stacking": { "group": "A", "mode": "normal" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" } }, "gridPos": { "h": 9, "w": 16, "x": 0, "y": 0 }, "id": 1, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (event) (rate(lilith_handler_dispatch_total{event=~\"$event\"}[5m]))", "legendFormat": "{{event}}", "range": true, "refId": "A" } ], "title": "Handler Dispatch Rate by Event", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Total dead-letter queue depth.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 10 }, { "color": "red", "value": 100 } ] }, "unit": "short" } }, "gridPos": { "h": 9, "w": 8, "x": 16, "y": 0 }, "id": 2, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "auto" }, "pluginVersion": "10.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(lilith_dlq_depth)", "legendFormat": "dlq depth", "range": true, "refId": "A" } ], "title": "Dead Letter Queue Depth", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Projection lag in seconds by projection name.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 20, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": true, "thresholdsStyle": { "mode": "line" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 10 }, { "color": "red", "value": 60 } ] }, "unit": "s" } }, "gridPos": { "h": 9, "w": 16, "x": 0, "y": 9 }, "id": 3, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "lilith_projection_lag_seconds{projection=~\"$projection\"}", "legendFormat": "{{projection}}", "range": true, "refId": "A" } ], "title": "Projection Lag", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "DLQ depth by event type.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "unit": "short" } }, "gridPos": { "h": 9, "w": 8, "x": 16, "y": 9 }, "id": 4, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": ["sum"], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Value" } ] }, "pluginVersion": "10.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (event) (lilith_dlq_depth)", "format": "table", "instant": true, "refId": "A" } ], "title": "DLQ by Event Type", "type": "table" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Handler processing duration percentiles.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 10, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": true }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 }, "id": 5, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.50, sum by (le, event) (rate(lilith_handler_duration_seconds_bucket{event=~\"$event\"}[5m])))", "legendFormat": "{{event}} p50", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "histogram_quantile(0.95, sum by (le, event) (rate(lilith_handler_duration_seconds_bucket{event=~\"$event\"}[5m])))", "legendFormat": "{{event}} p95", "range": true, "refId": "B" } ], "title": "Handler Duration Percentiles", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Handler failure rate (%).", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 25, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": true }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 }, { "color": "red", "value": 5 } ] }, "unit": "percent" } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 }, "id": 6, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "100 * sum by (event) (rate(lilith_handler_failures_total{event=~\"$event\"}[5m])) / clamp_min(sum by (event) (rate(lilith_handler_dispatch_total{event=~\"$event\"}[5m])), 0.001)", "legendFormat": "{{event}}", "range": true, "refId": "A" } ], "title": "Handler Failure Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Events retried per second by event type.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "bars", "fillOpacity": 80, "gradientMode": "none", "lineWidth": 1, "pointSize": 5, "showPoints": "never", "spanNulls": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 26 }, "id": 7, "options": { "legend": { "calcs": ["sum"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (event) (rate(lilith_handler_retries_total{event=~\"$event\"}[5m]))", "legendFormat": "{{event}}", "range": true, "refId": "A" } ], "title": "Handler Retries", "type": "timeseries" } ], "refresh": "30s", "schemaVersion": 39, "tags": ["oshun", "lilith", "events", "cqrs"], "templating": { "list": [ { "current": { "selected": false, "text": "Prometheus", "value": "prometheus" }, "hide": 0, "includeAll": false, "label": "Datasource", "multi": false, "name": "datasource", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allValue": ".+", "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(lilith_handler_dispatch_total, event)", "hide": 0, "includeAll": true, "label": "Event", "multi": true, "name": "event", "options": [], "query": { "query": "label_values(lilith_handler_dispatch_total, event)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" }, { "allValue": ".+", "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(lilith_projection_lag_seconds, projection)", "hide": 0, "includeAll": true, "label": "Projection", "multi": true, "name": "projection", "options": [], "query": { "query": "label_values(lilith_projection_lag_seconds, projection)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Lilith Event Handlers", "uid": "lilith-event-handlers", "version": 1, "weekStart": "" }