{
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": { "type": "grafana", "uid": "-- Grafana --" },
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "description": "Elasticsearch dashboard for Oshun (prometheus-community elasticsearch_exporter): cluster health, indexing/search throughput, latency, JVM heap, filesystem, pending tasks, shard counts, circuit breakers.",
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": null,
  "liveNow": false,
  "panels": [
    {
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "description": "0 = green, 1 = yellow, 2 = red.",
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "thresholds" },
          "mappings": [
            {
              "options": {
                "0": { "color": "green", "index": 0, "text": "GREEN" },
                "1": { "color": "orange", "index": 1, "text": "YELLOW" },
                "2": { "color": "red", "index": 2, "text": "RED" }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              { "color": "green", "value": null },
              { "color": "orange", "value": 1 },
              { "color": "red", "value": 2 }
            ]
          },
          "unit": "none"
        }
      },
      "gridPos": { "h": 4, "w": 6, "x": 0, "y": 0 },
      "id": 1,
      "options": {
        "colorMode": "background",
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
        "textMode": "value_and_name"
      },
      "pluginVersion": "10.2.0",
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${datasource}" },
          "editorMode": "code",
          "expr": "max(elasticsearch_cluster_health_status{color=\"red\"}) * 2 + max(elasticsearch_cluster_health_status{color=\"yellow\"}) * 1",
          "legendFormat": "health",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Cluster Health",
      "type": "stat"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "description": "Nodes currently in the cluster.",
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "thresholds" },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              { "color": "red", "value": null },
              { "color": "orange", "value": 1 },
              { "color": "green", "value": 3 }
            ]
          },
          "unit": "short"
        }
      },
      "gridPos": { "h": 4, "w": 6, "x": 6, "y": 0 },
      "id": 2,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
        "textMode": "value_and_name"
      },
      "pluginVersion": "10.2.0",
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${datasource}" },
          "editorMode": "code",
          "expr": "max(elasticsearch_cluster_health_number_of_nodes)",
          "legendFormat": "nodes",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Cluster Nodes",
      "type": "stat"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "description": "Total documents indexed across all primary shards.",
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "thresholds" },
          "mappings": [],
          "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] },
          "unit": "short"
        }
      },
      "gridPos": { "h": 4, "w": 6, "x": 12, "y": 0 },
      "id": 3,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
        "textMode": "value_and_name"
      },
      "pluginVersion": "10.2.0",
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${datasource}" },
          "editorMode": "code",
          "expr": "sum(elasticsearch_indices_docs)",
          "legendFormat": "docs",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Total Documents",
      "type": "stat"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "description": "Active shards (primary + replica).",
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "thresholds" },
          "mappings": [],
          "thresholds": { "mode": "absolute", "steps": [{ "color": "blue", "value": null }] },
          "unit": "short"
        }
      },
      "gridPos": { "h": 4, "w": 6, "x": 18, "y": 0 },
      "id": 4,
      "options": {
        "colorMode": "background",
        "graphMode": "area",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false },
        "textMode": "value_and_name"
      },
      "pluginVersion": "10.2.0",
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${datasource}" },
          "editorMode": "code",
          "expr": "max(elasticsearch_cluster_health_active_shards)",
          "legendFormat": "active",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Active Shards",
      "type": "stat"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "description": "Indexing and search throughput.",
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "never",
            "spanNulls": true
          },
          "mappings": [],
          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
          "unit": "ops"
        }
      },
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 },
      "id": 5,
      "options": {
        "legend": {
          "calcs": ["mean", "max"],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true
        },
        "tooltip": { "mode": "multi", "sort": "desc" }
      },
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${datasource}" },
          "editorMode": "code",
          "expr": "sum(rate(elasticsearch_indices_indexing_index_total[5m]))",
          "legendFormat": "index",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": { "type": "prometheus", "uid": "${datasource}" },
          "editorMode": "code",
          "expr": "sum(rate(elasticsearch_indices_search_query_total[5m]))",
          "legendFormat": "query",
          "range": true,
          "refId": "B"
        },
        {
          "datasource": { "type": "prometheus", "uid": "${datasource}" },
          "editorMode": "code",
          "expr": "sum(rate(elasticsearch_indices_search_fetch_total[5m]))",
          "legendFormat": "fetch",
          "range": true,
          "refId": "C"
        }
      ],
      "title": "Indexing & Search Throughput",
      "type": "timeseries"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "description": "Average indexing and search latency (total time / total ops).",
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "never",
            "spanNulls": true
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              { "color": "green", "value": null },
              { "color": "orange", "value": 0.1 },
              { "color": "red", "value": 1 }
            ]
          },
          "unit": "s"
        }
      },
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 },
      "id": 6,
      "options": {
        "legend": {
          "calcs": ["mean", "max"],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true
        },
        "tooltip": { "mode": "multi", "sort": "desc" }
      },
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${datasource}" },
          "editorMode": "code",
          "expr": "sum(rate(elasticsearch_indices_indexing_index_time_seconds_total[5m])) / clamp_min(sum(rate(elasticsearch_indices_indexing_index_total[5m])), 1)",
          "legendFormat": "avg index time",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": { "type": "prometheus", "uid": "${datasource}" },
          "editorMode": "code",
          "expr": "sum(rate(elasticsearch_indices_search_query_time_seconds_total[5m])) / clamp_min(sum(rate(elasticsearch_indices_search_query_total[5m])), 1)",
          "legendFormat": "avg query time",
          "range": true,
          "refId": "B"
        }
      ],
      "title": "Indexing & Search Average Latency",
      "type": "timeseries"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "description": "JVM heap used vs committed per node.",
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "never",
            "spanNulls": true
          },
          "mappings": [],
          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
          "unit": "bytes"
        }
      },
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 },
      "id": 7,
      "options": {
        "legend": {
          "calcs": ["mean", "max"],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true
        },
        "tooltip": { "mode": "multi", "sort": "desc" }
      },
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${datasource}" },
          "editorMode": "code",
          "expr": "sum by (name) (elasticsearch_jvm_memory_used_bytes{area=\"heap\"})",
          "legendFormat": "{{name}} used",
          "range": true,
          "refId": "A"
        },
        {
          "datasource": { "type": "prometheus", "uid": "${datasource}" },
          "editorMode": "code",
          "expr": "sum by (name) (elasticsearch_jvm_memory_committed_bytes{area=\"heap\"})",
          "legendFormat": "{{name}} committed",
          "range": true,
          "refId": "B"
        }
      ],
      "title": "JVM Heap (per node)",
      "type": "timeseries"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "description": "Filesystem utilisation per node.",
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "thresholds" },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "never",
            "spanNulls": true
          },
          "mappings": [],
          "max": 1,
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              { "color": "green", "value": null },
              { "color": "orange", "value": 0.75 },
              { "color": "red", "value": 0.9 }
            ]
          },
          "unit": "percentunit"
        }
      },
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 },
      "id": 8,
      "options": {
        "legend": {
          "calcs": ["lastNotNull", "max"],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true
        },
        "tooltip": { "mode": "multi", "sort": "desc" }
      },
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${datasource}" },
          "editorMode": "code",
          "expr": "1 - (elasticsearch_filesystem_data_available_bytes / clamp_min(elasticsearch_filesystem_data_size_bytes, 1))",
          "legendFormat": "{{name}} {{mount}}",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Filesystem Utilisation",
      "type": "timeseries"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "description": "Pending tasks in the cluster. Persistent non-zero is a sign of a slow master or a GC problem.",
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "thresholds" },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "never",
            "spanNulls": true
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              { "color": "green", "value": null },
              { "color": "orange", "value": 10 },
              { "color": "red", "value": 100 }
            ]
          },
          "unit": "short"
        }
      },
      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 },
      "id": 9,
      "options": {
        "legend": {
          "calcs": ["mean", "max"],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true
        },
        "tooltip": { "mode": "multi", "sort": "desc" }
      },
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${datasource}" },
          "editorMode": "code",
          "expr": "max(elasticsearch_cluster_health_number_of_pending_tasks)",
          "legendFormat": "pending",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Pending Cluster Tasks",
      "type": "timeseries"
    },
    {
      "datasource": { "type": "prometheus", "uid": "${datasource}" },
      "description": "Circuit-breaker trips per breaker. Non-zero means requests were refused to prevent OOM.",
      "fieldConfig": {
        "defaults": {
          "color": { "mode": "palette-classic" },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "opacity",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "never",
            "spanNulls": true
          },
          "mappings": [],
          "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] },
          "unit": "ops"
        }
      },
      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 },
      "id": 10,
      "options": {
        "legend": {
          "calcs": ["sum", "mean"],
          "displayMode": "table",
          "placement": "right",
          "showLegend": true
        },
        "tooltip": { "mode": "multi", "sort": "desc" }
      },
      "targets": [
        {
          "datasource": { "type": "prometheus", "uid": "${datasource}" },
          "editorMode": "code",
          "expr": "sum by (breaker) (rate(elasticsearch_breakers_tripped[10m]))",
          "legendFormat": "{{breaker}}",
          "range": true,
          "refId": "A"
        }
      ],
      "title": "Circuit Breaker Trips",
      "type": "timeseries"
    }
  ],
  "refresh": "30s",
  "schemaVersion": 39,
  "tags": ["oshun", "infrastructure", "elasticsearch", "search"],
  "templating": {
    "list": [
      {
        "current": { "selected": false, "text": "Prometheus", "value": "prometheus" },
        "hide": 0,
        "includeAll": false,
        "label": "Datasource",
        "multi": false,
        "name": "datasource",
        "options": [],
        "query": "prometheus",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "type": "datasource"
      }
    ]
  },
  "time": { "from": "now-3h", "to": "now" },
  "timepicker": {},
  "timezone": "",
  "title": "Infrastructure — Elasticsearch",
  "uid": "infra-elasticsearch",
  "version": 1,
  "weekStart": ""
}