{ "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "description": "Kafka dashboard for Oshun (danielqsj/kafka_exporter schema): broker count, messages in/out, consumer-group lag, partition leaders, under-replicated and offline partitions.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "liveNow": false, "panels": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Brokers alive in the cluster.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "orange", "value": 1 }, { "color": "green", "value": 3 } ] }, "unit": "short" } }, "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 }, "id": 1, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "value_and_name" }, "pluginVersion": "10.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "max(kafka_brokers)", "legendFormat": "brokers", "range": true, "refId": "A" } ], "title": "Brokers", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Distinct topics visible to the exporter.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] }, "unit": "short" } }, "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 }, "id": 2, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "value_and_name" }, "pluginVersion": "10.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "count(count by (topic) (kafka_topic_partitions))", "legendFormat": "topics", "range": true, "refId": "A" } ], "title": "Topics", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Total under-replicated partitions. Healthy clusters are at 0.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 1 }, { "color": "red", "value": 10 } ] }, "unit": "short" } }, "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 }, "id": 3, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "value_and_name" }, "pluginVersion": "10.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum(kafka_topic_partition_under_replicated_partition)", "legendFormat": "under-replicated", "range": true, "refId": "A" } ], "title": "Under-Replicated Partitions", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Total offline partitions. Offline partitions serve no traffic.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "unit": "short" } }, "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 }, "id": 4, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "textMode": "value_and_name" }, "pluginVersion": "10.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "count(kafka_topic_partition_leader == -1) or vector(0)", "legendFormat": "offline", "range": true, "refId": "A" } ], "title": "Offline Partitions", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Messages produced per second, top 10 topics.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": true }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "ops" } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, "id": 5, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "topk(10, sum by (topic) (rate(kafka_topic_partition_current_offset{topic=~\"$topic\"}[5m])))", "legendFormat": "{{topic}}", "range": true, "refId": "A" } ], "title": "Messages In (top 10 topics)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Consumer-group lag summed across partitions. Growing lag means consumers can't keep up.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": true }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 10000 }, { "color": "red", "value": 100000 } ] }, "unit": "short" } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, "id": 6, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (consumergroup) (kafka_consumergroup_lag{consumergroup=~\"$consumergroup\"})", "legendFormat": "{{consumergroup}}", "range": true, "refId": "A" } ], "title": "Consumer Group Lag", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Consumer group lag by topic — pinpoints the lagging subscription.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": true }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "short" } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, "id": 7, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "topk(10, sum by (consumergroup, topic) (kafka_consumergroup_lag{consumergroup=~\"$consumergroup\", topic=~\"$topic\"}))", "legendFormat": "{{consumergroup}} / {{topic}}", "range": true, "refId": "A" } ], "title": "Lag by (Group, Topic)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Consumer group message-consumption rate.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": true }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "ops" } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, "id": 8, "options": { "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "sum by (consumergroup) (rate(kafka_consumergroup_current_offset{consumergroup=~\"$consumergroup\"}[5m]))", "legendFormat": "{{consumergroup}}", "range": true, "refId": "A" } ], "title": "Consumer Group Throughput", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "Leader count per broker. Large imbalances hurt write throughput.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": true }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "short" } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 }, "id": 9, "options": { "legend": { "calcs": ["lastNotNull", "max"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "count by (broker) (kafka_topic_partition_leader == bool (on(broker) group_left() kafka_topic_partition_leader))", "legendFormat": "{{broker}}", "range": true, "refId": "A" } ], "title": "Leader Partitions per Broker", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "description": "In-sync replica count by topic — dips indicate replication trouble.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", "fillOpacity": 15, "gradientMode": "opacity", "lineInterpolation": "smooth", "lineWidth": 2, "pointSize": 5, "showPoints": "never", "spanNulls": true }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "short" } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 }, "id": 10, "options": { "legend": { "calcs": ["lastNotNull", "min"], "displayMode": "table", "placement": "right", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${datasource}" }, "editorMode": "code", "expr": "avg by (topic) (kafka_topic_partition_in_sync_replica{topic=~\"$topic\"})", "legendFormat": "{{topic}}", "range": true, "refId": "A" } ], "title": "Average ISR per Topic", "type": "timeseries" } ], "refresh": "30s", "schemaVersion": 39, "tags": ["oshun", "infrastructure", "kafka", "streaming"], "templating": { "list": [ { "current": { "selected": false, "text": "Prometheus", "value": "prometheus" }, "hide": 0, "includeAll": false, "label": "Datasource", "multi": false, "name": "datasource", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allValue": ".+", "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(kafka_topic_partitions, topic)", "hide": 0, "includeAll": true, "label": "Topic", "multi": true, "name": "topic", "options": [], "query": { "query": "label_values(kafka_topic_partitions, topic)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "^(?!__consumer_offsets$).*$", "skipUrlSync": false, "sort": 1, "type": "query" }, { "allValue": ".+", "current": { "selected": true, "text": ["All"], "value": ["$__all"] }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "definition": "label_values(kafka_consumergroup_lag, consumergroup)", "hide": 0, "includeAll": true, "label": "Consumer Group", "multi": true, "name": "consumergroup", "options": [], "query": { "query": "label_values(kafka_consumergroup_lag, consumergroup)", "refId": "StandardVariableQuery" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "type": "query" } ] }, "time": { "from": "now-3h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Infrastructure — Kafka", "uid": "infra-kafka", "version": 1, "weekStart": "" }