From 3774123827dd7df9a2459db194a5024797afc795 Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Sat, 14 Dec 2024 11:52:05 -0500 Subject: [PATCH] Refactor mixin dashboards (#885) This is a complete refactor of the dashboard system. It brings the dashboard creation, metrics, alerting, etc into alignment with other projects that use jsonnet/grafonnet/mixins. This should allow users to customize what we have created and deploy into their environments. The dashboard was the focus of this iteration, reaching parity with the previous dashboard. - Add in jsonnet and grafonnet - Add scripts to compile and lint mixin - Add CI for the mixin --------- Signed-off-by: Joe Adams --- .github/workflows/mixin.yml | 34 + .gitignore | 1 + elasticsearch-mixin/README.md | 29 + elasticsearch-mixin/compiled/alerts.yaml | 1 + .../compiled/dashboards/cluster.json | 687 +++++ elasticsearch-mixin/compiled/rules.yaml | 1 + elasticsearch-mixin/config.libsonnet | 6 + elasticsearch-mixin/dashboards.jsonnet | 3 + .../dashboards/cluster.libsonnet | 67 + .../dashboards/dashboards.libsonnet | 1 + elasticsearch-mixin/dashboards/g.libsonnet | 1 + .../dashboards/panels.libsonnet | 38 + .../dashboards/queries.libsonnet | 11 + .../dashboards/queries/document.libsonnet | 50 + .../dashboards/queries/general.libsonnet | 35 + .../dashboards/queries/memory.libsonnet | 47 + .../dashboards/queries/network.libsonnet | 28 + .../dashboards/queries/shard.libsonnet | 66 + .../dashboards/queries/threads.libsonnet | 24 + elasticsearch-mixin/dashboards/util.libsonnet | 66 + .../dashboards/variables.libsonnet | 15 + elasticsearch-mixin/jsonnetfile.json | 15 + elasticsearch-mixin/jsonnetfile.lock.json | 46 + elasticsearch-mixin/mixin.libsonnet | 3 + examples/grafana/dashboard.json | 2250 ----------------- scripts/compile-mixin.sh | 8 + scripts/lint-jsonnet.sh | 13 + 27 files changed, 1296 insertions(+), 2250 deletions(-) create mode 100644 .github/workflows/mixin.yml create mode 100644 elasticsearch-mixin/README.md create mode 100644 elasticsearch-mixin/compiled/alerts.yaml create mode 100644 elasticsearch-mixin/compiled/dashboards/cluster.json create mode 100644 elasticsearch-mixin/compiled/rules.yaml create mode 100644 elasticsearch-mixin/config.libsonnet create mode 100644 elasticsearch-mixin/dashboards.jsonnet create mode 100644 elasticsearch-mixin/dashboards/cluster.libsonnet create mode 100644 elasticsearch-mixin/dashboards/dashboards.libsonnet create mode 100644 elasticsearch-mixin/dashboards/g.libsonnet create mode 100644 elasticsearch-mixin/dashboards/panels.libsonnet create mode 100644 elasticsearch-mixin/dashboards/queries.libsonnet create mode 100644 elasticsearch-mixin/dashboards/queries/document.libsonnet create mode 100644 elasticsearch-mixin/dashboards/queries/general.libsonnet create mode 100644 elasticsearch-mixin/dashboards/queries/memory.libsonnet create mode 100644 elasticsearch-mixin/dashboards/queries/network.libsonnet create mode 100644 elasticsearch-mixin/dashboards/queries/shard.libsonnet create mode 100644 elasticsearch-mixin/dashboards/queries/threads.libsonnet create mode 100644 elasticsearch-mixin/dashboards/util.libsonnet create mode 100644 elasticsearch-mixin/dashboards/variables.libsonnet create mode 100644 elasticsearch-mixin/jsonnetfile.json create mode 100644 elasticsearch-mixin/jsonnetfile.lock.json create mode 100644 elasticsearch-mixin/mixin.libsonnet delete mode 100644 examples/grafana/dashboard.json create mode 100755 scripts/compile-mixin.sh create mode 100755 scripts/lint-jsonnet.sh diff --git a/.github/workflows/mixin.yml b/.github/workflows/mixin.yml new file mode 100644 index 00000000..2c282069 --- /dev/null +++ b/.github/workflows/mixin.yml @@ -0,0 +1,34 @@ +--- +name: mixin +on: + pull_request: + paths: + - "elasticsearch-mixin/**" + +jobs: + check-mixin: + name: check + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: 1.22.5 + - name: Install dependencies + run: | + go install github.com/google/go-jsonnet/cmd/jsonnet@v0.20.0 + go install github.com/google/go-jsonnet/cmd/jsonnetfmt@v0.20.0 + go install github.com/google/go-jsonnet/cmd/jsonnet-lint@v0.20.0 + go install github.com/monitoring-mixins/mixtool/cmd/mixtool@16dc166166d91e93475b86b9355a4faed2400c18 + go install github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@v0.5.1 + - name: Lint + run: bash ./scripts/lint-jsonnet.sh + - name: Compile mixin + run: bash ./scripts/compile-mixin.sh + - name: Verify compiled mixin matches repo + run: | + git diff --exit-code -- ./elasticsearch-mixin || (echo "Compiled mixin does not match repo" && exit 1) + # Check if there are any new untracked files + test -z "$(git status --porcelain)" || (echo "Untracked files found, please run ./scripts/compile-mixin.sh" && exit 1) diff --git a/.gitignore b/.gitignore index e470da31..9687f990 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ elasticsearch_exporter *-stamp .tarballs /vendor +vendor/ diff --git a/elasticsearch-mixin/README.md b/elasticsearch-mixin/README.md new file mode 100644 index 00000000..4f41eb2d --- /dev/null +++ b/elasticsearch-mixin/README.md @@ -0,0 +1,29 @@ +# Elasticsearch Exporter Mixin + +This is a mixin for the elasticsearch_exporter to define dashboards, alerts, and monitoring queries for use with this exporter. + +Good example of upstream mixin for reference: https://github.com/kubernetes-monitoring/kubernetes-mixin + +## Development + +### JSONNET +https://jsonnet.org/ + +```go install github.com/google/go-jsonnet/cmd/jsonnet@latest``` + +### JSONNET BUNDLER +jsonnet bundler is a package manager for jsonnet + +https://github.com/jsonnet-bundler/jsonnet-bundler + +```go install -a github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@latest``` + +### Grafonnet +Grafana libraries for jsonnet: https://grafana.github.io/grafonnet/ + +```jb install github.com/grafana/grafonnet/gen/grafonnet-latest@main``` + +### Run the build +```bash +./scripts/compile-mixin.sh +``` diff --git a/elasticsearch-mixin/compiled/alerts.yaml b/elasticsearch-mixin/compiled/alerts.yaml new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/elasticsearch-mixin/compiled/alerts.yaml @@ -0,0 +1 @@ +{} diff --git a/elasticsearch-mixin/compiled/dashboards/cluster.json b/elasticsearch-mixin/compiled/dashboards/cluster.json new file mode 100644 index 00000000..37c919b6 --- /dev/null +++ b/elasticsearch-mixin/compiled/dashboards/cluster.json @@ -0,0 +1,687 @@ +{ + "graphTooltip": 1, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [ ], + "title": "Overview", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 2, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_number_of_nodes{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 3, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_number_of_data_nodes{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Data Nodes", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 4, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_number_of_pending_tasks{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Pending Tasks", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 5, + "panels": [ ], + "title": "Shards", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 6 + }, + "id": 6, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_active_shards{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Active", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 6 + }, + "id": 7, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_active_primary_shards{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Active Primary", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 6 + }, + "id": 8, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_initializing_shards{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Initializing", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 6 + }, + "id": 9, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_reloacting_shards{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Relocating", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 6 + }, + "id": 10, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_unassigned_shards{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "Unassigned", + "type": "stat" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 20, + "y": 6 + }, + "id": 11, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(\n elasticsearch_cluster_health_delayed_unassigned_shards{cluster=~\"$cluster\"}\n)\n" + } + ], + "title": "DelayedUnassigned", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 12, + "panels": [ ], + "title": "Documents", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 11 + }, + "id": 13, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "elasticsearch_indices_docs{cluster=~\"$cluster\"}\n" + } + ], + "title": "Indexed Documents", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 11 + }, + "id": 14, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "elasticsearch_indices_store_size_bytes{cluster=~\"$cluster\"}\n" + } + ], + "title": "Index Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 11 + }, + "id": 15, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(elasticsearch_indices_indexing_index_total{cluster=~\"$cluster\"}[$__rate_interval])\n", + "legendFormat": "{{name}}" + } + ], + "title": "Index Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 11 + }, + "id": 16, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(elasticsearch_indices_search_query_total{cluster=~\"$cluster\"}[$__rate_interval])\n", + "legendFormat": "{{name}}" + } + ], + "title": "Query Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 11 + }, + "id": 17, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "sum(elasticsearch_thread_pool_queue_count{cluster=~\"$cluster\",type!=\"management\"}) by (type)\n", + "legendFormat": "{{type}}" + } + ], + "title": "Queue Count", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 18, + "panels": [ ], + "title": "Memory", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 16 + }, + "id": 19, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "elasticsearch_jvm_memory_used_bytes{cluster=~\"$cluster\"}\n", + "legendFormat": "{{name}} {{area}}" + } + ], + "title": "Memory Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "max": 1, + "min": 0, + "unit": "percentunit" + } + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 16 + }, + "id": 20, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "avg_over_time(\n elasticsearch_jvm_memory_used_bytes{cluster=~\"$cluster\"}[15m]\n) /\nelasticsearch_jvm_memory_max_bytes{cluster=~\"$cluster\"}\n", + "legendFormat": "{{name}} {{area}}" + } + ], + "title": "Memory 15m Avg", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 16 + }, + "id": 21, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "elasticsearch_jvm_memory_max_bytes{cluster=~\"$cluster\"}\n", + "legendFormat": "{{name}} {{area}}" + } + ], + "title": "Memory Max", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 16 + }, + "id": 22, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(\n elasticsearch_jvm_gc_collection_seconds_sum{cluster=~\"$cluster\"}[$__rate_interval]\n)\n", + "legendFormat": "{{name}} {{gc}}" + } + ], + "title": "GC Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 23, + "panels": [ ], + "title": "Threads", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 24, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "elasticsearch_thread_pool_active_count{cluster=~\"$cluster\"}\n", + "legendFormat": "{{type}}" + } + ], + "title": "Thread Pools", + "type": "timeseries" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 25, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "elasticsearch_thread_pool_rejected_count{cluster=~\"$cluster\"}\n", + "legendFormat": "{{name}} {{type}}" + } + ], + "title": "Thread Pool Rejections", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 26, + "panels": [ ], + "title": "Network", + "type": "row" + }, + { + "datasource": { + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 27, + "pluginVersion": "v10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(\n elasticsearch_transport_rx_size_bytes_total{cluster=~\"$cluster\"}[$__rate_interval]\n)\n", + "legendFormat": "{{name}} TX" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "expr": "rate(\n elasticsearch_transport_tx_size_bytes_total{cluster=~\"$cluster\"}[$__rate_interval]\n)\n", + "legendFormat": "{{name}} RX" + } + ], + "title": "Transport Rate", + "type": "timeseries" + } + ], + "refresh": "1m", + "schemaVersion": 36, + "tags": [ + "elasticsearch-exporter-mixin" + ], + "templating": { + "list": [ + { + "name": "datasource", + "query": "prometheus", + "type": "datasource" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "name": "cluster", + "query": "label_values(elasticsearch_cluster_health_status, cluster)", + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timezone": "utc", + "title": "Elasticsearch Exporter / Cluster" + } \ No newline at end of file diff --git a/elasticsearch-mixin/compiled/rules.yaml b/elasticsearch-mixin/compiled/rules.yaml new file mode 100644 index 00000000..0967ef42 --- /dev/null +++ b/elasticsearch-mixin/compiled/rules.yaml @@ -0,0 +1 @@ +{} diff --git a/elasticsearch-mixin/config.libsonnet b/elasticsearch-mixin/config.libsonnet new file mode 100644 index 00000000..3cf4a4f9 --- /dev/null +++ b/elasticsearch-mixin/config.libsonnet @@ -0,0 +1,6 @@ +{ + _config+:: { + dashboardNamePrefix: 'Elasticsearch Exporter / ', + dashboardTags: ['elasticsearch-exporter-mixin'], + }, +} diff --git a/elasticsearch-mixin/dashboards.jsonnet b/elasticsearch-mixin/dashboards.jsonnet new file mode 100644 index 00000000..9a03074e --- /dev/null +++ b/elasticsearch-mixin/dashboards.jsonnet @@ -0,0 +1,3 @@ +local dashboards = (import 'mixin.libsonnet').grafanaDashboards; + +{ [name]: dashboards[name] for name in std.objectFields(dashboards) } diff --git a/elasticsearch-mixin/dashboards/cluster.libsonnet b/elasticsearch-mixin/dashboards/cluster.libsonnet new file mode 100644 index 00000000..ac32d75e --- /dev/null +++ b/elasticsearch-mixin/dashboards/cluster.libsonnet @@ -0,0 +1,67 @@ +local g = import 'g.libsonnet'; + +local dashboard = g.dashboard; +local row = g.panel.row; + +local panels = import './panels.libsonnet'; +local queries = import './queries.libsonnet'; +local variables = import './variables.libsonnet'; +local util = import './util.libsonnet'; + +{ + grafanaDashboards+:: { + 'cluster.json': + dashboard.new('%s Cluster' % $._config.dashboardNamePrefix) + + dashboard.withTags($._config.dashboardTags) + + dashboard.withRefresh('1m') + + dashboard.time.withFrom(value='now-1h') + + dashboard.graphTooltip.withSharedCrosshair() + + dashboard.withVariables([ + variables.datasource, + variables.cluster, + ]) + + dashboard.withPanels( + util.makeGrid([ + row.new('Overview') + + row.withPanels([ + panels.stat.nodes('Nodes', queries.runningNodes), + panels.stat.nodes('Data Nodes', queries.dataNodes), + panels.stat.nodes('Pending Tasks', queries.pendingTasks), + ]), + row.new('Shards') + + row.withPanels([ + panels.stat.nodes('Active', queries.activeShards), + panels.stat.nodes('Active Primary', queries.activePrimaryShards), + panels.stat.nodes('Initializing', queries.initializingShards), + panels.stat.nodes('Relocating', queries.reloactingShards), + panels.stat.nodes('Unassigned', queries.unassignedShards), + panels.stat.nodes('DelayedUnassigned', queries.delayedUnassignedShards), + ]), + row.new('Documents') + + row.withPanels([ + panels.timeSeries.base('Indexed Documents', queries.indexedDocuments), + panels.timeSeries.bytes('Index Size', queries.indexSize), + panels.timeSeries.base('Index Rate', queries.indexRate), + panels.timeSeries.base('Query Rate', queries.queryRate), + panels.timeSeries.base('Queue Count', queries.queueCount), + ]), + row.new('Memory') + + row.withPanels([ + panels.timeSeries.bytes('Memory Usage', queries.memoryUsage), + panels.timeSeries.ratioMax1('Memory 15m Avg', queries.memoryUsageAverage15), + panels.timeSeries.bytes('Memory Max', queries.memoryMax), + panels.timeSeries.seconds('GC Rate', queries.gcSeconds), + ]), + row.new('Threads') + + row.withPanels([ + panels.timeSeries.base('Thread Pools', queries.threadPoolActive), + panels.timeSeries.base('Thread Pool Rejections', queries.threadPoolRejections), + ]), + row.new('Network') + + row.withPanels([ + panels.timeSeries.bytes('Transport Rate', [queries.transportTXRate, queries.transportRXRate]), + ]), + ]), + ), + }, +} diff --git a/elasticsearch-mixin/dashboards/dashboards.libsonnet b/elasticsearch-mixin/dashboards/dashboards.libsonnet new file mode 100644 index 00000000..16802c25 --- /dev/null +++ b/elasticsearch-mixin/dashboards/dashboards.libsonnet @@ -0,0 +1 @@ +(import 'cluster.libsonnet') diff --git a/elasticsearch-mixin/dashboards/g.libsonnet b/elasticsearch-mixin/dashboards/g.libsonnet new file mode 100644 index 00000000..69aac830 --- /dev/null +++ b/elasticsearch-mixin/dashboards/g.libsonnet @@ -0,0 +1 @@ +import 'github.com/grafana/grafonnet/gen/grafonnet-latest/main.libsonnet' diff --git a/elasticsearch-mixin/dashboards/panels.libsonnet b/elasticsearch-mixin/dashboards/panels.libsonnet new file mode 100644 index 00000000..f5f24d45 --- /dev/null +++ b/elasticsearch-mixin/dashboards/panels.libsonnet @@ -0,0 +1,38 @@ +local g = import 'g.libsonnet'; + +{ + stat: { + local stat = g.panel.stat, + + base(title, targets): + stat.new(title) + + stat.queryOptions.withTargets(targets), + + nodes: self.base, + }, + + timeSeries: { + local timeSeries = g.panel.timeSeries, + + base(title, targets): + timeSeries.new(title) + + timeSeries.queryOptions.withTargets(targets), + + ratio(title, targets): + self.base(title, targets) + + timeSeries.standardOptions.withUnit('percentunit'), + + ratioMax1(title, targets): + self.ratio(title, targets) + + timeSeries.standardOptions.withMax(1) + + timeSeries.standardOptions.withMin(0), + + bytes(title, targets): + self.base(title, targets) + + timeSeries.standardOptions.withUnit('bytes'), + + seconds(title, targets): + self.base(title, targets) + + timeSeries.standardOptions.withUnit('s'), + }, +} diff --git a/elasticsearch-mixin/dashboards/queries.libsonnet b/elasticsearch-mixin/dashboards/queries.libsonnet new file mode 100644 index 00000000..d7a8e661 --- /dev/null +++ b/elasticsearch-mixin/dashboards/queries.libsonnet @@ -0,0 +1,11 @@ +local g = import './g.libsonnet'; +local prometheusQuery = g.query.prometheus; + +local variables = import './variables.libsonnet'; + +(import './queries/general.libsonnet') + +(import './queries/shard.libsonnet') + +(import './queries/document.libsonnet') + +(import './queries/memory.libsonnet') + +(import './queries/threads.libsonnet') + +(import './queries/network.libsonnet') diff --git a/elasticsearch-mixin/dashboards/queries/document.libsonnet b/elasticsearch-mixin/dashboards/queries/document.libsonnet new file mode 100644 index 00000000..369ce885 --- /dev/null +++ b/elasticsearch-mixin/dashboards/queries/document.libsonnet @@ -0,0 +1,50 @@ +local g = import '../g.libsonnet'; +local prometheusQuery = g.query.prometheus; + +local variables = import '../variables.libsonnet'; + +{ + indexedDocuments: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + elasticsearch_indices_docs{cluster=~"$cluster"} + ||| + ), + + indexSize: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + elasticsearch_indices_store_size_bytes{cluster=~"$cluster"} + ||| + ), + + indexRate: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + rate(elasticsearch_indices_indexing_index_total{cluster=~"$cluster"}[$__rate_interval]) + ||| + ) + + prometheusQuery.withLegendFormat('{{name}}'), + + queryRate: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + rate(elasticsearch_indices_search_query_total{cluster=~"$cluster"}[$__rate_interval]) + ||| + ) + + prometheusQuery.withLegendFormat('{{name}}'), + + queueCount: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum(elasticsearch_thread_pool_queue_count{cluster=~"$cluster",type!="management"}) by (type) + ||| + ) + + prometheusQuery.withLegendFormat('{{type}}'), + +} diff --git a/elasticsearch-mixin/dashboards/queries/general.libsonnet b/elasticsearch-mixin/dashboards/queries/general.libsonnet new file mode 100644 index 00000000..053207ac --- /dev/null +++ b/elasticsearch-mixin/dashboards/queries/general.libsonnet @@ -0,0 +1,35 @@ +local g = import '../g.libsonnet'; +local prometheusQuery = g.query.prometheus; + +local variables = import '../variables.libsonnet'; + +{ + runningNodes: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_number_of_nodes{cluster=~"$cluster"} + ) + ||| + ), + dataNodes: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_number_of_data_nodes{cluster=~"$cluster"} + ) + ||| + ), + + pendingTasks: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_number_of_pending_tasks{cluster=~"$cluster"} + ) + ||| + ), +} diff --git a/elasticsearch-mixin/dashboards/queries/memory.libsonnet b/elasticsearch-mixin/dashboards/queries/memory.libsonnet new file mode 100644 index 00000000..59e8d274 --- /dev/null +++ b/elasticsearch-mixin/dashboards/queries/memory.libsonnet @@ -0,0 +1,47 @@ +local g = import '../g.libsonnet'; +local prometheusQuery = g.query.prometheus; + +local variables = import '../variables.libsonnet'; + +{ + memoryUsage: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + elasticsearch_jvm_memory_used_bytes{cluster=~"$cluster"} + ||| + ) + + prometheusQuery.withLegendFormat('{{name}} {{area}}'), + + memoryUsageAverage15: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + avg_over_time( + elasticsearch_jvm_memory_used_bytes{cluster=~"$cluster"}[15m] + ) / + elasticsearch_jvm_memory_max_bytes{cluster=~"$cluster"} + ||| + ) + + prometheusQuery.withLegendFormat('{{name}} {{area}}'), + + memoryMax: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + elasticsearch_jvm_memory_max_bytes{cluster=~"$cluster"} + ||| + ) + + prometheusQuery.withLegendFormat('{{name}} {{area}}'), + + gcSeconds: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + rate( + elasticsearch_jvm_gc_collection_seconds_sum{cluster=~"$cluster"}[$__rate_interval] + ) + ||| + ) + + prometheusQuery.withLegendFormat('{{name}} {{gc}}'), +} diff --git a/elasticsearch-mixin/dashboards/queries/network.libsonnet b/elasticsearch-mixin/dashboards/queries/network.libsonnet new file mode 100644 index 00000000..76aab2b5 --- /dev/null +++ b/elasticsearch-mixin/dashboards/queries/network.libsonnet @@ -0,0 +1,28 @@ +local g = import '../g.libsonnet'; +local prometheusQuery = g.query.prometheus; + +local variables = import '../variables.libsonnet'; + +{ + transportTXRate: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + rate( + elasticsearch_transport_rx_size_bytes_total{cluster=~"$cluster"}[$__rate_interval] + ) + ||| + ) + + prometheusQuery.withLegendFormat('{{name}} TX'), + + transportRXRate: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + rate( + elasticsearch_transport_tx_size_bytes_total{cluster=~"$cluster"}[$__rate_interval] + ) + ||| + ) + + prometheusQuery.withLegendFormat('{{name}} RX'), +} diff --git a/elasticsearch-mixin/dashboards/queries/shard.libsonnet b/elasticsearch-mixin/dashboards/queries/shard.libsonnet new file mode 100644 index 00000000..6bebb4e6 --- /dev/null +++ b/elasticsearch-mixin/dashboards/queries/shard.libsonnet @@ -0,0 +1,66 @@ +local g = import '../g.libsonnet'; +local prometheusQuery = g.query.prometheus; + +local variables = import '../variables.libsonnet'; + +{ + activeShards: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_active_shards{cluster=~"$cluster"} + ) + ||| + ), + + activePrimaryShards: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_active_primary_shards{cluster=~"$cluster"} + ) + ||| + ), + + initializingShards: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_initializing_shards{cluster=~"$cluster"} + ) + ||| + ), + + reloactingShards: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_reloacting_shards{cluster=~"$cluster"} + ) + ||| + ), + + unassignedShards: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_unassigned_shards{cluster=~"$cluster"} + ) + ||| + ), + + delayedUnassignedShards: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + sum( + elasticsearch_cluster_health_delayed_unassigned_shards{cluster=~"$cluster"} + ) + ||| + ), +} diff --git a/elasticsearch-mixin/dashboards/queries/threads.libsonnet b/elasticsearch-mixin/dashboards/queries/threads.libsonnet new file mode 100644 index 00000000..1fdb3e63 --- /dev/null +++ b/elasticsearch-mixin/dashboards/queries/threads.libsonnet @@ -0,0 +1,24 @@ +local g = import '../g.libsonnet'; +local prometheusQuery = g.query.prometheus; + +local variables = import '../variables.libsonnet'; + +{ + threadPoolActive: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + elasticsearch_thread_pool_active_count{cluster=~"$cluster"} + ||| + ) + + prometheusQuery.withLegendFormat('{{type}}'), + + threadPoolRejections: + prometheusQuery.new( + '$' + variables.datasource.name, + ||| + elasticsearch_thread_pool_rejected_count{cluster=~"$cluster"} + ||| + ) + + prometheusQuery.withLegendFormat('{{name}} {{type}}'), +} diff --git a/elasticsearch-mixin/dashboards/util.libsonnet b/elasticsearch-mixin/dashboards/util.libsonnet new file mode 100644 index 00000000..388b06ae --- /dev/null +++ b/elasticsearch-mixin/dashboards/util.libsonnet @@ -0,0 +1,66 @@ +local g = import 'g.libsonnet'; +local panelUtil = g.util.panel; + +{ + local gridWidth = 24, + + // makeGrid returns an array of panels organized into a grid layout. + // This is a modified version of the grafonnet makeGrid function to + // calculate the width of each panel based on the number of panels. + makeGrid(panels, panelHeight=4, startY=0): + local sanitizePanels(ps) = + // Figure out the number of panels and the width of each panel + local numPanels = std.length(ps); + local panelWidth = std.floor(gridWidth / numPanels); + + // Sanitize the panels, this ensures tht the panels have the valid gridPos + std.map( + function(p) + local sanePanel = panelUtil.sanitizePanel(p, defaultHeight=panelHeight); + ( + if p.type == 'row' + then sanePanel { + panels: sanitizePanels(sanePanel.panels), + } + else sanePanel { + gridPos+: { + w: panelWidth, + }, + } + ), + ps + ); + + local sanitizedPanels = sanitizePanels(panels); + + local grouped = panelUtil.groupPanelsInRows(sanitizedPanels); + + local panelsBeforeRows = panelUtil.getPanelsBeforeNextRow(grouped); + local rowPanels = + std.filter( + function(p) p.type == 'row', + grouped + ); + + + local CalculateXforPanel(index, panel) = + local panelsPerRow = std.floor(gridWidth / panel.gridPos.w); + local col = std.mod(index, panelsPerRow); + panel { gridPos+: { x: panel.gridPos.w * col } }; + + + local panelsBeforeRowsWithX = std.mapWithIndex(CalculateXforPanel, panelsBeforeRows); + + local rowPanelsWithX = + std.map( + function(row) + row { panels: std.mapWithIndex(CalculateXforPanel, row.panels) }, + rowPanels + ); + + local uncollapsed = panelUtil.resolveCollapsedFlagOnRows(panelsBeforeRowsWithX + rowPanelsWithX); + + local normalized = panelUtil.normalizeY(uncollapsed); + + std.map(function(p) p { gridPos+: { y+: startY } }, normalized), +} diff --git a/elasticsearch-mixin/dashboards/variables.libsonnet b/elasticsearch-mixin/dashboards/variables.libsonnet new file mode 100644 index 00000000..53233a23 --- /dev/null +++ b/elasticsearch-mixin/dashboards/variables.libsonnet @@ -0,0 +1,15 @@ +local g = import './g.libsonnet'; +local var = g.dashboard.variable; + +{ + datasource: + var.datasource.new('datasource', 'prometheus'), + + cluster: + var.query.new('cluster') + + var.query.withDatasourceFromVariable(self.datasource) + + var.query.queryTypes.withLabelValues( + 'cluster', + 'elasticsearch_cluster_health_status', + ), +} diff --git a/elasticsearch-mixin/jsonnetfile.json b/elasticsearch-mixin/jsonnetfile.json new file mode 100644 index 00000000..2414c867 --- /dev/null +++ b/elasticsearch-mixin/jsonnetfile.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/grafana/grafonnet.git", + "subdir": "gen/grafonnet-latest" + } + }, + "version": "main" + } + ], + "legacyImports": true +} diff --git a/elasticsearch-mixin/jsonnetfile.lock.json b/elasticsearch-mixin/jsonnetfile.lock.json new file mode 100644 index 00000000..31b59c22 --- /dev/null +++ b/elasticsearch-mixin/jsonnetfile.lock.json @@ -0,0 +1,46 @@ +{ + "version": 1, + "dependencies": [ + { + "source": { + "git": { + "remote": "https://github.com/grafana/grafonnet.git", + "subdir": "gen/grafonnet-latest" + } + }, + "version": "1c56af39815c4903e47c27194444456f005f65df", + "sum": "GxEO83uxgsDclLp/fmlUJZDbSGpeUZY6Ap3G2cgdL1g=" + }, + { + "source": { + "git": { + "remote": "https://github.com/grafana/grafonnet.git", + "subdir": "gen/grafonnet-v10.4.0" + } + }, + "version": "1c56af39815c4903e47c27194444456f005f65df", + "sum": "DKj+Sn+rlI48g/aoJpzkfPge46ya0jLk5kcZoiZ2X/I=" + }, + { + "source": { + "git": { + "remote": "https://github.com/jsonnet-libs/docsonnet.git", + "subdir": "doc-util" + } + }, + "version": "6ac6c69685b8c29c54515448eaca583da2d88150", + "sum": "BrAL/k23jq+xy9oA7TWIhUx07dsA/QLm3g7ktCwe//U=" + }, + { + "source": { + "git": { + "remote": "https://github.com/jsonnet-libs/xtd.git", + "subdir": "" + } + }, + "version": "63d430b69a95741061c2f7fc9d84b1a778511d9c", + "sum": "qiZi3axUSXCVzKUF83zSAxklwrnitMmrDK4XAfjPMdE=" + } + ], + "legacyImports": false +} diff --git a/elasticsearch-mixin/mixin.libsonnet b/elasticsearch-mixin/mixin.libsonnet new file mode 100644 index 00000000..7083cb16 --- /dev/null +++ b/elasticsearch-mixin/mixin.libsonnet @@ -0,0 +1,3 @@ +// (import 'alerts/alerts.libsonnet') + +(import 'dashboards/dashboards.libsonnet') + +(import 'config.libsonnet') diff --git a/examples/grafana/dashboard.json b/examples/grafana/dashboard.json deleted file mode 100644 index 6b1de5a8..00000000 --- a/examples/grafana/dashboard.json +++ /dev/null @@ -1,2250 +0,0 @@ -{ - "__inputs": [], - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "5.4.0" - }, - { - "type": "panel", - "id": "graph", - "name": "Graph", - "version": "5.0.0" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "5.0.0" - }, - { - "type": "panel", - "id": "singlestat", - "name": "Singlestat", - "version": "5.0.0" - } - ], - "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": "-- Grafana --", - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] - }, - "editable": true, - "gnetId": null, - "graphTooltip": 1, - "id": null, - "iteration": 1549021227642, - "links": [], - "panels": [ - { - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 0 - }, - "id": 90, - "title": "Cluster", - "type": "row" - }, - { - "cacheTimeout": null, - "colorBackground": true, - "colorPostfix": false, - "colorPrefix": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 92, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": false - }, - "tableColumn": "Value", - "targets": [ - { - "expr": "scalar(elasticsearch_cluster_health_status{color=\"green\",cluster=~\"$cluster\"}) + scalar(elasticsearch_cluster_health_status{color=\"yellow\",cluster=~\"$cluster\"}) * 2 + scalar(elasticsearch_cluster_health_status{color=\"red\",cluster=~\"$cluster\"}) * 3", - "format": "time_series", - "instant": false, - "intervalFactor": 1, - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": "2,3", - "title": "Cluster Status", - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - }, - { - "op": "=", - "text": "green", - "value": "1" - }, - { - "op": "=", - "text": "yellow", - "value": "2" - }, - { - "op": "=", - "text": "red", - "value": "3" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "$server", - "editable": true, - "error": false, - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 1 - }, - "id": 8, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(elasticsearch_cluster_health_number_of_nodes{cluster=~\"$cluster\"})/count(elasticsearch_cluster_health_number_of_nodes{cluster=~\"$cluster\"})", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "", - "metric": "elasticsearch_cluster_health_number_of_nodes", - "refId": "A", - "step": 1800 - } - ], - "thresholds": "", - "timeFrom": null, - "timeShift": null, - "title": "Running Nodes", - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 1 - }, - "id": 94, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "elasticsearch_cluster_health_number_of_data_nodes{cluster=~\"$cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Active Data Nodes", - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 1 - }, - "id": 96, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "elasticsearch_cluster_health_number_of_pending_tasks{cluster=~\"$cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "0.5,1", - "title": "Pending Tasks", - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 4 - }, - "id": 76, - "panels": [], - "title": "Shards", - "type": "row" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 0, - "y": 5 - }, - "id": 78, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "elasticsearch_cluster_health_active_shards{cluster=~\"$cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Active Shards", - "transparent": false, - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 4, - "y": 5 - }, - "id": 80, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "elasticsearch_cluster_health_active_primary_shards{cluster=~\"$cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "", - "title": "Active Primary Shards", - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 8, - "y": 5 - }, - "id": 82, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "elasticsearch_cluster_health_initializing_shards{cluster=~\"$cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "0.5,1", - "title": "Initializing Shards", - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 12, - "y": 5 - }, - "id": 84, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "elasticsearch_cluster_health_relocating_shards{cluster=~\"$cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "0.5,1", - "title": "Relocating Shards", - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 16, - "y": 5 - }, - "id": 86, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "elasticsearch_cluster_health_unassigned_shards{cluster=~\"$cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "0.5,1", - "title": "Unassigned Shards", - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": true, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], - "format": "none", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 4, - "x": 20, - "y": 5 - }, - "id": 88, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "elasticsearch_cluster_health_delayed_unassigned_shards{cluster=~\"$cluster\"}", - "format": "time_series", - "intervalFactor": 1, - "refId": "A" - } - ], - "thresholds": "0.5,1", - "title": "Delayed Unassigned Shards", - "type": "singlestat", - "valueFontSize": "120%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "id": 70, - "panels": [], - "title": "Documents", - "type": "row" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$server", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 3, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(elasticsearch_indices_docs{cluster=~\"$cluster\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Documents", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Documents indexed", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$server", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 4, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(elasticsearch_indices_store_size_bytes{cluster=~\"$cluster\"})", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Index Size", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Index Size", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 72, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(elasticsearch_indices_indexing_index_total{cluster=~\"$cluster\"}[1h])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Documents Indexed Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Documents/s", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 74, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(elasticsearch_indices_search_fetch_total{cluster=~\"$cluster\"}[1h])", - "format": "time_series", - "intervalFactor": 1, - "legendFormat": "{{name}}", - "refId": "A" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Query Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "Queris/s", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$server", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 24, - "x": 0, - "y": 27 - }, - "height": "", - "id": 64, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(elasticsearch_thread_pool_queue_count{cluster=~\"$cluster\", type!=\"management\"}) by (type)", - "interval": "", - "intervalFactor": 2, - "legendFormat": "Type: {{type}}", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Queue Count", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 35 - }, - "id": 68, - "panels": [], - "title": "System", - "type": "row" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "$server", - "editable": true, - "error": false, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 0, - "y": 36 - }, - "height": "", - "id": 12, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(node_memory_MemTotal_bytes{cluster=~\"$cluster\"})", - "intervalFactor": 2, - "legendFormat": "", - "metric": "", - "refId": "A", - "step": 1800 - } - ], - "thresholds": "", - "title": "Total Memory", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "$server", - "editable": true, - "error": false, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 8, - "y": 36 - }, - "height": "", - "id": 13, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(node_memory_MemFree_bytes{cluster=~\"$cluster\"})", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 1800 - } - ], - "thresholds": "", - "title": "Total Memory Free", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "cacheTimeout": null, - "colorBackground": false, - "colorValue": false, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "datasource": "$server", - "editable": true, - "error": false, - "format": "bytes", - "gauge": { - "maxValue": 100, - "minValue": 0, - "show": false, - "thresholdLabels": false, - "thresholdMarkers": true - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 16, - "y": 36 - }, - "height": "", - "id": 14, - "interval": null, - "links": [], - "mappingType": 1, - "mappingTypes": [ - { - "name": "value to text", - "value": 1 - }, - { - "name": "range to text", - "value": 2 - } - ], - "maxDataPoints": 100, - "nullPointMode": "connected", - "nullText": null, - "postfix": "", - "postfixFontSize": "50%", - "prefix": "", - "prefixFontSize": "50%", - "rangeMaps": [ - { - "from": "null", - "text": "N/A", - "to": "null" - } - ], - "sparkline": { - "fillColor": "rgba(31, 118, 189, 0.18)", - "full": false, - "lineColor": "rgb(31, 120, 193)", - "show": true - }, - "tableColumn": "", - "targets": [ - { - "expr": "sum(node_memory_MemAvailable_bytes{cluster=~\"$cluster\"})", - "intervalFactor": 2, - "legendFormat": "", - "refId": "A", - "step": 1800 - } - ], - "thresholds": "", - "title": "Total Memory Available", - "type": "singlestat", - "valueFontSize": "80%", - "valueMaps": [ - { - "op": "=", - "text": "N/A", - "value": "null" - } - ], - "valueName": "current" - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$server", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 39 - }, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(elasticsearch_thread_pool_active_count{cluster=~\"$cluster\", type!=\"management\"}) by (type)", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "Type: {{ type }}", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Thread Pools", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$server", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 66, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(elasticsearch_thread_pool_rejected_count{cluster=~\"$cluster\", type!=\"management\"}[5m])", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{ name }} {{ type }}", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Thread pool rejections", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$server", - "fill": 1, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 47 - }, - "id": 57, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "bucketAggs": [ - { - "id": "2", - "settings": { - "interval": "auto" - }, - "type": "date_histogram" - } - ], - "dsType": "elasticsearch", - "expr": "avg(irate(node_cpu_seconds_total{cluster=~\"$cluster\"}[60s])) by(mode) *100", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ mode }}", - "metric": "elasticsearch_breakers_tripped", - "metrics": [ - { - "id": "1", - "type": "count" - } - ], - "refId": "A", - "step": 240, - "timeField": "failure_tstamp" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Avg. CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": "0", - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$server", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 47 - }, - "id": 28, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "rightSide": false, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "avg_over_time(elasticsearch_jvm_memory_used_bytes{area=\"heap\",cluster=~\"$cluster\"}[15m]) / elasticsearch_jvm_memory_max_bytes{area=\"heap\",cluster=~\"$cluster\"}", - "format": "time_series", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{ name }}", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "Avg Heap in 15min", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": "", - "logBase": 1, - "max": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$server", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 56 - }, - "id": 5, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "max": true, - "min": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(elasticsearch_transport_rx_packets_total{cluster=~\"$cluster\"}[5m]))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "RX", - "refId": "A", - "step": 240 - }, - { - "expr": "sum(rate(elasticsearch_transport_tx_packets_total{cluster=~\"$cluster\"}[5m])) * -1", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "TX", - "refId": "B", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "RX/TX Rate 5m", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$server", - "editable": true, - "error": false, - "fill": 1, - "grid": {}, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 56 - }, - "id": 65, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "connected", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "irate(elasticsearch_jvm_gc_collection_seconds_sum{cluster=~\"$cluster\"}[1m])", - "interval": "", - "intervalFactor": 2, - "legendFormat": "{{ name }} {{ gc }}", - "refId": "A", - "step": 240 - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "GC seconds", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - } - ], - "refresh": "1m", - "schemaVersion": 16, - "style": "dark", - "tags": [ - "infra", - "b2c", - "elastic" - ], - "templating": { - "list": [ - { - "current": { - "text": "Prometheus", - "value": "Prometheus" - }, - "hide": 0, - "label": "Server", - "name": "server", - "options": [], - "query": "prometheus", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "type": "datasource" - }, - { - "allValue": null, - "current": {}, - "datasource": "$server", - "definition": "", - "hide": 0, - "includeAll": true, - "label": null, - "multi": true, - "name": "cluster", - "options": [], - "query": "label_values(elasticsearch_cluster_health_status,cluster)", - "refresh": 1, - "regex": "", - "skipUrlSync": false, - "sort": 0, - "tagValuesQuery": "", - "tags": [], - "tagsQuery": "", - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-12h", - "to": "now" - }, - "timepicker": { - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" - ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "utc", - "title": "Elasticsearch", - "uid": "n_nxrE_mk", - "version": 2 -} diff --git a/scripts/compile-mixin.sh b/scripts/compile-mixin.sh new file mode 100755 index 00000000..1cd1f849 --- /dev/null +++ b/scripts/compile-mixin.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +MIXIN_PATH=./elasticsearch-mixin +MIXIN_OUT_PATH=./elasticsearch-mixin/compiled + +rm -rf ${MIXIN_OUT_PATH} && mkdir ${MIXIN_OUT_PATH} +pushd ${MIXIN_PATH} && jb install && popd +mixtool generate all --output-alerts ${MIXIN_OUT_PATH}/alerts.yaml --output-rules ${MIXIN_OUT_PATH}/rules.yaml --directory ${MIXIN_OUT_PATH}/dashboards ${MIXIN_PATH}/mixin.libsonnet diff --git a/scripts/lint-jsonnet.sh b/scripts/lint-jsonnet.sh new file mode 100755 index 00000000..74348bf5 --- /dev/null +++ b/scripts/lint-jsonnet.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +# Run lint on all jsonnet files in the repository +RESULT=0; +for f in $(find . -name 'vendor' -prune -o -name '*.libsonnet' -print -o -name '*.jsonnet' -print); do + # jsonnet fmt -i "$$f" + echo "Linting ${f}" + jsonnetfmt -- "${f}" | diff -u "${f}" - + RESULT=$((RESULT+$?)) +done + +echo "Linting complete" +exit $RESULT