From 9bd3d7bd3ab227f76be8ff388e086da50cb1d300 Mon Sep 17 00:00:00 2001 From: Eugene Klimov Date: Wed, 8 Jan 2025 15:45:17 +0500 Subject: [PATCH] remove runningDifference from grafana dashboards to avoid errors in modern clickhouse versions, upgrade plugin versions (#1609) remove runningDifference from grafana dashboards --- .../grafana-dashboard-operator-cr-template.yaml | 4 ++-- .../grafana-dashboard-queries-cr-template.yaml | 2 +- .../files/ClickHouse_Queries_dashboard.json | 6 +++--- grafana-dashboard/ClickHouse_Queries_dashboard.json | 6 +++--- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/deploy/grafana/grafana-with-grafana-operator/grafana-dashboard-operator-cr-template.yaml b/deploy/grafana/grafana-with-grafana-operator/grafana-dashboard-operator-cr-template.yaml index 8c3464771..6dc2246e2 100644 --- a/deploy/grafana/grafana-with-grafana-operator/grafana-dashboard-operator-cr-template.yaml +++ b/deploy/grafana/grafana-with-grafana-operator/grafana-dashboard-operator-cr-template.yaml @@ -14,6 +14,6 @@ spec: # TODO remove this plugin definition after resolve https://github.com/integr8ly/grafana-operator/issues/155 plugins: - name: "vertamedia-clickhouse-datasource" - version: "2.5.4" + version: "3.3.1" - name: "grafana-piechart-panel" - version: "1.6.2" + version: "1.6.4" diff --git a/deploy/grafana/grafana-with-grafana-operator/grafana-dashboard-queries-cr-template.yaml b/deploy/grafana/grafana-with-grafana-operator/grafana-dashboard-queries-cr-template.yaml index a0f4f9a31..fcbb96762 100644 --- a/deploy/grafana/grafana-with-grafana-operator/grafana-dashboard-queries-cr-template.yaml +++ b/deploy/grafana/grafana-with-grafana-operator/grafana-dashboard-queries-cr-template.yaml @@ -13,4 +13,4 @@ spec: datasourceName: "$GRAFANA_PROMETHEUS_DATASOURCE_NAME" plugins: - name: "vertamedia-clickhouse-datasource" - version: "2.5.4" + version: "3.3.1" diff --git a/deploy/helm/clickhouse-operator/files/ClickHouse_Queries_dashboard.json b/deploy/helm/clickhouse-operator/files/ClickHouse_Queries_dashboard.json index f348932a8..ee27a17ed 100644 --- a/deploy/helm/clickhouse-operator/files/ClickHouse_Queries_dashboard.json +++ b/deploy/helm/clickhouse-operator/files/ClickHouse_Queries_dashboard.json @@ -147,8 +147,8 @@ "format": "time_series", "interval": "", "intervalFactor": 2, - "query": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND type IN ($type), $type)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT $top\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", - "rawQuery": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n \r\n \r\n \r\n \r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n \r\n \r\n \r\n \r\n \r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT 30\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", + "query": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / (t/1000 - lagInFrame(t/1000,1,0) OVER ()) ), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND type IN ($type), $type)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT $top\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", + "rawQuery": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / (t/1000 - lagInFrame(t/1000,1,0) OVER ()) ), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n \r\n \r\n \r\n \r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n \r\n \r\n \r\n \r\n \r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT 30\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", "refId": "A", "resultFormat": "time_series", "round": "0s", @@ -743,7 +743,7 @@ "interval": "", "intervalFactor": 2, "query": "$rate(count() c)\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\n", - "rawQuery": "SELECT t, c/runningDifference(t/1000) cRate FROM ( SELECT (intDiv(toUInt32(event_time), 4) * 4) * 1000 AS t, count() c FROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829) AND event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829)\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n \n \n \n GROUP BY t ORDER BY t)", + "rawQuery": "SELECT t, c/(t/1000 - lagInFrame(t/1000,1,0) OVER ()) cRate FROM ( SELECT (intDiv(toUInt32(event_time), 4) * 4) * 1000 AS t, count() c FROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829) AND event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829)\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n \n \n \n GROUP BY t ORDER BY t)", "refId": "A", "resultFormat": "time_series", "round": "0s", diff --git a/grafana-dashboard/ClickHouse_Queries_dashboard.json b/grafana-dashboard/ClickHouse_Queries_dashboard.json index 20e080270..a349c6017 100644 --- a/grafana-dashboard/ClickHouse_Queries_dashboard.json +++ b/grafana-dashboard/ClickHouse_Queries_dashboard.json @@ -147,8 +147,8 @@ "format": "time_series", "interval": "", "intervalFactor": 2, - "query": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND type IN ($type), $type)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT $top\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", - "rawQuery": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / runningDifference(t / 1000)), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n \r\n \r\n \r\n \r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n \r\n \r\n \r\n \r\n \r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT 30\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", + "query": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / (t/1000 - lagInFrame(t/1000,1,0) OVER ()) ), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE $timeFilter\r\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\r\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\r\n $conditionalTest(AND type IN ($type), $type)\r\n $conditionalTest(AND initial_user IN ($user), $user)\r\n $conditionalTest(AND query_duration_ms >= $min_duration_ms, $min_duration_ms)\r\n $conditionalTest(AND query_duration_ms <= $max_duration_ms, $max_duration_ms)\r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT $top\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", + "rawQuery": "SELECT\r\n t,\r\n arrayMap(a -> (a.1, a.2 / (t/1000 - lagInFrame(t/1000,1,0) OVER ()) ), groupArr)\r\nFROM (\r\n SELECT t, groupArray((q, c)) AS groupArr\r\n FROM (\r\n SELECT\r\n (intDiv(toUInt32(event_time), 2) * 2) * 1000 AS t,\r\n normalizeQuery(query) AS q,\r\n count() c\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n \r\n \r\n \r\n \r\n AND normalized_query_hash GLOBAL IN (\r\n SELECT normalized_query_hash AS h\r\n FROM cluster('all-sharded',system.query_log)\r\n WHERE event_date >= toDate(1694531137) AND event_date <= toDate(1694534737) AND event_time >= toDateTime(1694531137) AND event_time <= toDateTime(1694534737)\r\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\r\n \r\n \r\n \r\n \r\n \r\n GROUP BY h\r\n ORDER BY count() DESC\r\n LIMIT 30\r\n SETTINGS skip_unavailable_shards=1\r\n )\r\n GROUP BY t, query\r\n ORDER BY t\r\n )\r\n GROUP BY t\r\n ORDER BY t\r\n) SETTINGS skip_unavailable_shards=1", "refId": "A", "resultFormat": "time_series", "round": "0s", @@ -743,7 +743,7 @@ "interval": "", "intervalFactor": 2, "query": "$rate(count() c)\nFROM cluster('all-sharded',system.query_log)\nWHERE $timeFilter\n AND( ('$type' = '1,2,3,4' AND type != 'QueryStart') OR ('$type' != '1,2,3,4' AND type IN ($type)))\n $conditionalTest(AND query_kind IN ($query_kind), $query_kind)\n $conditionalTest(AND initial_user IN ($user), $user)\n $conditionalTest(AND query_duration_ms >= $min_duration_ms,$min_duration_ms)\n $conditionalTest(AND query_duration_ms <= $max_duration_ms,$max_duration_ms)\n", - "rawQuery": "SELECT t, c/runningDifference(t/1000) cRate FROM ( SELECT (intDiv(toUInt32(event_time), 4) * 4) * 1000 AS t, count() c FROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829) AND event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829)\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n \n \n \n GROUP BY t ORDER BY t)", + "rawQuery": "SELECT t, c/(t/1000 - lagInFrame(t/1000,1,0) OVER ()) cRate FROM ( SELECT (intDiv(toUInt32(event_time), 4) * 4) * 1000 AS t, count() c FROM cluster('all-sharded',system.query_log)\nWHERE event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829) AND event_date >= toDate(1694531229) AND event_date <= toDate(1694534829) AND event_time >= toDateTime(1694531229) AND event_time <= toDateTime(1694534829)\n AND( ('1,2,3,4' = '1,2,3,4' AND type != 'QueryStart') OR ('1,2,3,4' != '1,2,3,4' AND type IN (1,2,3,4)))\n \n \n \n GROUP BY t ORDER BY t)", "refId": "A", "resultFormat": "time_series", "round": "0s",