Skip to content

Instantly share code, notes, and snippets.

@kgoralski

kgoralski/ClickHouse_By_Namespace.json

Last active November 12, 2023 20:28
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save kgoralski/357f9b5d14d7a40e1e95347febb1eb56 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save kgoralski/357f9b5d14d7a40e1e95347febb1eb56 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
ClickHouse_By_Namespace.json
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"enable": true,
"expr": "resets(ClickHouseAsyncMetrics_Uptime{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\", pod=~\"$pod\"}[$__rate_interval])",
"hide": false,
"iconColor": "rgba(255, 96, 96, 1)",
"name": "Restarts",
"showIn": 0,
"step": "60s",
"tagKeys": "instance",
"titleFormat": "Restart",
"useValueForTime": false
}
]
},
"description": "ClickHouse internal exporter metrics",
"editable": true,
"fiscalYearStartMonth": 0,
"gnetId": 14192,
"graphTooltip": 1,
"id": 38,
"links": [
{
"asDropdown": false,
"icon": "external link",
"includeVars": false,
"keepTime": false,
"tags": [],
"targetBlank": true,
"title": "ClickHouse monitoring docs",
"tooltip": "",
"type": "link",
"url": "https://clickhouse.tech/docs/en/operations/monitoring/"
}
],
"liveNow": false,
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"align": "auto",
"displayMode": "auto",
"filterable": false,
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "uptime"
},
"properties": [
{
"id": "unit",
"value": "s"
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 24,
"x": 0,
"y": 0
},
"id": 14,
"options": {
"footer": {
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"frameIndex": 1,
"showHeader": true
},
"pluginVersion": "9.3.8",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_VersionInteger{namespace=~\"$namespace\", pod=~\"$pod\"}",
"format": "table",
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_Revision{namespace=~\"$namespace\", pod=~\"$pod\"}",
"format": "table",
"hide": false,
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseAsyncMetrics_NumberOfTables{namespace=~\"$namespace\", pod=~\"$pod\"}",
"format": "table",
"hide": false,
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "C"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseAsyncMetrics_Uptime{namespace=~\"$namespace\", pod=~\"$pod\"}",
"format": "table",
"hide": false,
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "D"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseAsyncMetrics_NumberOfDatabases{namespace=~\"$namespace\", pod=~\"$pod\"}",
"format": "table",
"hide": false,
"instant": true,
"interval": "",
"legendFormat": "",
"refId": "E"
}
],
"title": "Nodes",
"transformations": [
{
"id": "seriesToColumns",
"options": {
"byField": "instance"
}
},
{
"id": "organize",
"options": {
"excludeByName": {
"Time 1": true,
"Time 10": true,
"Time 2": true,
"Time 3": true,
"Time 4": true,
"Time 5": true,
"Time 6": true,
"Time 7": true,
"Time 8": true,
"Time 9": true,
"Value #A 2": true,
"Value #B 2": true,
"Value #C 2": true,
"Value #D 2": true,
"Value #E 2": true,
"__name__ 1": true,
"__name__ 10": true,
"__name__ 2": true,
"__name__ 3": true,
"__name__ 4": true,
"__name__ 5": true,
"__name__ 6": true,
"__name__ 7": true,
"__name__ 8": true,
"__name__ 9": true,
"job 1": true,
"job 10": true,
"job 2": true,
"job 3": true,
"job 4": true,
"job 5": true,
"job 6": true,
"job 7": true,
"job 8": true,
"job 9": true
},
"indexByName": {
"Time 1": 6,
"Time 10": 37,
"Time 2": 9,
"Time 3": 13,
"Time 4": 16,
"Time 5": 20,
"Time 6": 23,
"Time 7": 27,
"Time 8": 30,
"Time 9": 34,
"Value #A 1": 1,
"Value #A 2": 12,
"Value #B 1": 2,
"Value #B 2": 19,
"Value #C 1": 3,
"Value #C 2": 26,
"Value #D 1": 5,
"Value #D 2": 33,
"Value #E 1": 4,
"Value #E 2": 40,
"__name__ 1": 7,
"__name__ 10": 38,
"__name__ 2": 10,
"__name__ 3": 14,
"__name__ 4": 17,
"__name__ 5": 21,
"__name__ 6": 24,
"__name__ 7": 28,
"__name__ 8": 31,
"__name__ 9": 35,
"instance": 0,
"job 1": 8,
"job 10": 39,
"job 2": 11,
"job 3": 15,
"job 4": 18,
"job 5": 22,
"job 6": 25,
"job 7": 29,
"job 8": 32,
"job 9": 36
},
"renameByName": {
"Value #A 1": "version",
"Value #B 1": "revision",
"Value #C 1": "tables",
"Value #D 1": "uptime",
"Value #E 1": "databases"
}
}
}
],
"transparent": true,
"type": "table"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Total amount of memory (bytes) allocated by the server.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 5,
"w": 12,
"x": 0,
"y": 8
},
"hiddenSeries": false,
"id": 29,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_MemoryTracking{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Memory",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "bytes",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of connections to TCP server (clients with native interface), also included server-server distributed query connections. \nNumber of connections to HTTP server",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 5,
"w": 12,
"x": 12,
"y": 8
},
"hiddenSeries": false,
"id": 73,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_TCPConnection{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "tcp - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_HTTPConnection{namespace=~\"$namespace\", pod=~\"$pod\"}",
"hide": false,
"interval": "",
"legendFormat": "http - {{pod}}",
"refId": "B"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Connections",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of times the number of query processing threads was lowered due to slow reads.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 5,
"w": 12,
"x": 0,
"y": 13
},
"hiddenSeries": false,
"id": 72,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBackoff{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ReadBackoff{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Read backoff",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of reads from a file that were slow. This indicate system overload. Thresholds are controlled by read_backoff_* settings.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 5,
"w": 12,
"x": 12,
"y": 13
},
"hiddenSeries": false,
"id": 71,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_SlowRead{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_SlowRead{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Slow reads",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"collapsed": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 18
},
"id": 12,
"panels": [],
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A"
}
],
"title": "Queries",
"type": "row"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 19
},
"hiddenSeries": false,
"id": 2,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_Query{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_Query{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Queries",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Same as Queries, but only for SELECT queries.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 19
},
"hiddenSeries": false,
"id": 3,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectQuery{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_SelectQuery{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "B"
}
],
"thresholds": [],
"timeRegions": [],
"title": "SELECT queries",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Same as Queries, but only for INSERT queries.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 27
},
"hiddenSeries": false,
"id": 4,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_InsertQuery{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_InsertQuery{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "B"
}
],
"thresholds": [],
"timeRegions": [],
"title": "INSERT queries",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Same as Failed queries, but only for SELECT queries.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 27
},
"hiddenSeries": false,
"id": 6,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_FailedSelectQuery{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_FailedSelectQuery{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "B"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Failed SELECT queries",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of failed queries.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 35
},
"hiddenSeries": false,
"id": 5,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_FailedQuery{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_FailedQuery{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "B"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Failed queries",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of times when memory limit exceeded for query.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 35
},
"hiddenSeries": false,
"id": 85,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_QueryMemoryLimitExceeded{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_QueryMemoryLimitExceeded{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "B"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Queries memory limit",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of queries that are stopped and waiting due to 'priority' setting.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 43
},
"hiddenSeries": false,
"id": 86,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseMetrics_QueryPreempted{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseMetrics_QueryPreempted{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "B"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Query preemted",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Same as Failed queries, but only for INSERT queries.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 43
},
"hiddenSeries": false,
"id": 7,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_FailedInsertQuery{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_FailedInsertQuery{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "B"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Failed INSERT queries",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Avg queries latencies",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 51
},
"hiddenSeries": false,
"id": 8,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null as zero",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "increase(ClickHouseProfileEvents_QueryTimeMicroseconds{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_Query{instance=~\"$instance\"}[$__rate_interval])",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Query latencies (avg)",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "µs",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": 1
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Avg SELECT queries latencies",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 51
},
"hiddenSeries": false,
"id": 9,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null as zero",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "increase(ClickHouseProfileEvents_SelectQueryTimeMicroseconds{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_SelectQuery{instance=~\"$instance\"}[$__rate_interval])",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "SELECT query latencies (avg)",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "µs",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Avg INSERT queries latencies",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 59
},
"hiddenSeries": false,
"id": 10,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null as zero",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "increase(ClickHouseProfileEvents_InsertQueryTimeMicroseconds{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_InsertQuery{instance=~\"$instance\"}[$__rate_interval])",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "INSERT query latencies (avg)",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "µs",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"collapsed": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 67
},
"id": 64,
"panels": [],
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A"
}
],
"title": "Insert",
"type": "row"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of rows INSERTed to all tables.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 68
},
"hiddenSeries": false,
"id": 62,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_InsertedRows{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_InsertedRows{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Inserted rows",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of bytes (uncompressed; for columns as they stored in memory) INSERTed to all tables.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 68
},
"hiddenSeries": false,
"id": 65,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_InsertedBytes{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_InsertedBytes{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Inserted bytes",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "binBps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of times the INSERT of a block to a MergeTree table was throttled due to high number of active data parts for partition.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 76
},
"hiddenSeries": false,
"id": 66,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_DelayedInserts{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_DelayedInserts{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Delayed inserts",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of times the INSERT of a block to a MergeTree table was rejected with 'Too many parts' exception due to high number of active data parts for partition.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 76
},
"hiddenSeries": false,
"id": 67,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_RejectedInserts{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_RejectedInserts{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Rejected inserts",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Total number of milliseconds spent while the INSERT of a block to a MergeTree table was throttled due to high number of active data parts for partition.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 84
},
"hiddenSeries": false,
"id": 69,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null as zero",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "increase(ClickHouseProfileEvents_DelayedInsertsMilliseconds{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_DelayedInserts{instance=~\"$instance\"}[$__rate_interval])",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Delayed inserts blocks (avg)",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "µs",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": 1
}
},
{
"collapsed": true,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 92
},
"id": 125,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of data parts selected to read from a MergeTree table.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 27
},
"hiddenSeries": false,
"id": 126,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedParts{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_SelectedParts{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Parts",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of (non-adjacent) ranges in all data parts selected to read from a MergeTree table.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 27
},
"hiddenSeries": false,
"id": 127,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedRanges{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_SelectedRanges{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Ranges",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of marks (index granules) selected to read from a MergeTree table.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 35
},
"hiddenSeries": false,
"id": 128,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedMarks{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_SelectedMarks{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Marks",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of rows SELECTed from all tables.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 35
},
"hiddenSeries": false,
"id": 129,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedRows{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_SelectedRows{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Rows",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of bytes (uncompressed; for columns as they stored in memory) SELECTed from all tables.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 43
},
"hiddenSeries": false,
"id": 130,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_SelectedBytes{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_SelectedBytes{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Bytes",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "binBps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
}
],
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A"
}
],
"title": "Select",
"type": "row"
},
{
"collapsed": true,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 93
},
"id": 17,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of times the 'lseek' function was called.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 33
},
"hiddenSeries": false,
"id": 20,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_Seek{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_Seek{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "lseek",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "ops",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of files opened",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 33
},
"hiddenSeries": false,
"id": 19,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_FileOpen{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_FileOpen{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Open Files",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "ops",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of reads (read/pread) from a file descriptor. Does not include sockets.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 41
},
"hiddenSeries": false,
"id": 22,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorRead{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorRead{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Read from FD",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "ops",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of writes (write/pwrite) to a file descriptor. Does not include sockets.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 41
},
"hiddenSeries": false,
"id": 23,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWrite{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWrite{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Write to FD",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "ops",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of times the read (read/pread) from a file descriptor have failed.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 49
},
"hiddenSeries": false,
"id": 24,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadFailed{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadFailed{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Read from FD failed",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "ops",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of times the read (read/pread) from a file descriptor have failed.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 49
},
"hiddenSeries": false,
"id": 25,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadFailed{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadFailed{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Read from FD failed",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "ops",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of bytes read from file descriptors. If the file is compressed, this will show the compressed data size.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 57
},
"hiddenSeries": false,
"id": 21,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadBytes{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ReadBufferFromFileDescriptorReadBytes{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Read bytes from FD",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "binBps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of bytes written to file descriptors. If the file is compressed, this will show compressed data size.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 57
},
"hiddenSeries": false,
"id": 26,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWriteBytes{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_WriteBufferFromFileDescriptorWriteBytes{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Write bytes to FD",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "binBps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of bytes (the number of bytes before decompression) read from compressed sources (files, network).",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 65
},
"hiddenSeries": false,
"id": 51,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ReadCompressedBytes{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ReadCompressedBytes{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Read compressed bytes",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "binBps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of uncompressed bytes (the number of bytes after decompression) read from compressed sources (files, network).",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 65
},
"hiddenSeries": false,
"id": 53,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_CompressedReadBufferBytes{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_CompressedReadBufferBytes{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Read uncompressed bytes",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "binBps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of compressed blocks (the blocks of data that are compressed independent of each other) read from compressed sources (files, network).",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 73
},
"hiddenSeries": false,
"id": 52,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_CompressedReadBufferBlocks{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_CompressedReadBufferBlocks{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Compressed blocks",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
}
],
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A"
}
],
"title": "IO",
"type": "row"
},
{
"collapsed": true,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 94
},
"id": 56,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of Replicated tables that are currently in readonly state due to re-initialization after ZooKeeper session loss or due to startup without ZooKeeper configured.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 31
},
"hiddenSeries": false,
"id": 74,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_ReadonlyReplica{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Readonly replica",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of times a data part was failed to download from replica of a ReplicatedMergeTree table.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 31
},
"hiddenSeries": false,
"id": 58,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartFailedFetches{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ReplicatedPartFailedFetches{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Replicated part failed fetches",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of times a data part was downloaded from replica of a ReplicatedMergeTree table.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 39
},
"hiddenSeries": false,
"id": 57,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartFetches{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ReplicatedPartFetches{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Replicated part fetches",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of times we prefer to download already merged part from replica of ReplicatedMergeTree table instead of performing a merge ourself (usually we prefer doing a merge ourself to save network traffic). This happens when we have not all source parts to perform a merge or when the data part is old enough.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 39
},
"hiddenSeries": false,
"id": 60,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartFetchesOfMerged{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ReplicatedPartFetchesOfMerged{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Replicated part fetches or merges",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of times data parts of ReplicatedMergeTree tables were successfully merged.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 47
},
"hiddenSeries": false,
"id": 59,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedPartMerges{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ReplicatedPartMerges{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Replicated part merges",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of times a data part that we wanted doesn't exist on any replica (even on replicas that are offline right now). That data parts are definitely lost. This is normal due to asynchronous replication (if quorum inserts were not enabled), when the replica on which the data part was written was failed and when it became online after fail it doesn't contain that data part.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 47
},
"hiddenSeries": false,
"id": 61,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ReplicatedDataLoss{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ReplicatedDataLoss{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Replicated part data loss",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
}
],
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A"
}
],
"title": "Replicas",
"type": "row"
},
{
"collapsed": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 95
},
"id": 76,
"panels": [],
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A"
}
],
"title": "Merge",
"type": "row"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of launched background merges.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 96
},
"hiddenSeries": false,
"id": 77,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_Merge{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_Merge{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Merge",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Rows read for background merges. This is the number of rows before merge.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 96
},
"hiddenSeries": false,
"id": 78,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_MergedRows{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_MergedRows{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Merged rows",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Uncompressed bytes (for columns as they stored in memory) that was read for background merges. This is the number before merge.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 104
},
"hiddenSeries": false,
"id": 79,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_MergedUncompressedBytes{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_MergedUncompressedBytes{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Merge uncompressed bytes",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "binBps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Avg merge duration",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 104
},
"hiddenSeries": false,
"id": 80,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null as zero",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "increase(ClickHouseProfileEvents_MergesTimeMilliseconds{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_Merge{instance=~\"$instance\"}[$__rate_interval])",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Merge duration (avg)",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "µs",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": 1
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of rows INSERTed to MergeTree tables.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 112
},
"hiddenSeries": false,
"id": 81,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterRows{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_MergeTreeDataWriterRows{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "MergeTree rows",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of blocks INSERTed to MergeTree tables. Each block forms a data part of level zero.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 112
},
"hiddenSeries": false,
"id": 82,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "9.3.8",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterBlocks{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_MergeTreeDataWriterBlocks{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "MergeTree blocks",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Uncompressed bytes (for columns as they stored in memory) INSERTed to MergeTree tables.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 120
},
"hiddenSeries": false,
"id": 83,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterUncompressedBytes{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_MergeTreeDataWriterUncompressedBytes{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "MergeTree uncompressed bytes",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "binBps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Bytes written to filesystem for data INSERTed to MergeTree tables.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 120
},
"hiddenSeries": false,
"id": 84,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_MergeTreeDataWriterCompressedBytes{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_MergeTreeDataWriterCompressedBytes{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "MergeTree compressed bytes",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "binBps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Disk space reserved for currently running background merges. It is slightly more than the total size of currently merging parts.",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 128
},
"hiddenSeries": false,
"id": 123,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_DiskSpaceReservedForMerge{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Reserved space",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "bytes",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"collapsed": true,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 136
},
"id": 92,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 26
},
"hiddenSeries": false,
"id": 102,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_UncompressedCacheHits{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_UncompressedCacheHits{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Uncompressed Cache Hits",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 26
},
"hiddenSeries": false,
"id": 103,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_UncompressedCacheMisses{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_UncompressedCacheMisses{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Uncompressed Cache Misses",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 34
},
"hiddenSeries": false,
"id": 104,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_MarkCacheHits{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_MarkCacheHits{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Mark Cache Hits",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 34
},
"hiddenSeries": false,
"id": 105,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_MarkCacheMisses{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_MarkCacheMisses{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Mark Cache Misses",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
}
],
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A"
}
],
"title": "Cache",
"type": "row"
},
{
"collapsed": true,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 137
},
"id": 90,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "The part is generating now, it is not in data_parts list.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 27
},
"hiddenSeries": false,
"id": 114,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_PartsTemporary{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Temporary",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "The part is in data_parts, but not used for SELECTs.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 27
},
"hiddenSeries": false,
"id": 115,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_PartsPreCommitted{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Pre commited",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "The part is in data_parts, but not used for SELECTs.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 35
},
"hiddenSeries": false,
"id": 116,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_PartsCommitted{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Commited",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Not active data part, but could be used by only current SELECTs, could be deleted after SELECTs finishes.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 35
},
"hiddenSeries": false,
"id": 117,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_PartsOutdated{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Outdated",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Not active data part with identity refcounter, it is deleting right now by a cleaner.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 43
},
"hiddenSeries": false,
"id": 118,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_PartsDeleting{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Deleting",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Part was moved to another disk and should be deleted in own destructor.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 43
},
"hiddenSeries": false,
"id": 119,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_PartsDeleteOnDestroy{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Delete on destroy",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Wide parts.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 51
},
"hiddenSeries": false,
"id": 120,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_PartsWide{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Wide",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Compact parts.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 51
},
"hiddenSeries": false,
"id": 122,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_PartsCompact{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Compact",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "In-memory parts.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 59
},
"hiddenSeries": false,
"id": 121,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_PartsInMemory{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "In-memory",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
}
],
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A"
}
],
"title": "Parts",
"type": "row"
},
{
"collapsed": true,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 138
},
"id": 94,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of connections to remote servers sending data that was INSERTed into Distributed tables. Both synchronous and asynchronous mode.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 25
},
"hiddenSeries": false,
"id": 112,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_DistributedSend{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Send",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of pending files to process for asynchronous insertion into Distributed tables. Number of files for every shard is summed.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 25
},
"hiddenSeries": false,
"id": 113,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_DistributedFilesToInsert{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Files to insert",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of times the INSERT of a block to a Distributed table was throttled due to high number of pending bytes.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 33
},
"hiddenSeries": false,
"id": 106,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedDelayedInserts{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_DistributedDelayedInserts{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Distributed delayed inserts",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of times the INSERT of a block to a Distributed table was rejected with 'Too many bytes' exception due to high number of pending bytes.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 33
},
"hiddenSeries": false,
"id": 107,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedRejectedInserts{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_DistributedRejectedInserts{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Distributed rejected inserts",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Total number of milliseconds spent while the INSERT of a block to a Distributed table was throttled due to high number of pending bytes.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 41
},
"hiddenSeries": false,
"id": 108,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null as zero",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "increase(ClickHouseProfileEvents_DistributedDelayedInsertsMilliseconds{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) / increase(ClickHouseProfileEvents_DistributedDelayedInserts{instance=~\"$instance\"}[$__rate_interval])",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Distributed delayed inserts blocks (avg)",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "µs",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": 1
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Total count when distributed connection fails with retry",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 41
},
"hiddenSeries": false,
"id": 109,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedConnectionFailTry{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_DistributedConnectionFailTry{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Distributed connection fail try",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Total count when distributed connection fails after all retries finished",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 49
},
"hiddenSeries": false,
"id": 110,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedConnectionFailAtAll{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_DistributedConnectionFailAtAll{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Distributed connection fail at all",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 49
},
"hiddenSeries": false,
"id": 111,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_DistributedSyncInsertionTimeoutExceeded{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_DistributedSyncInsertionTimeoutExceeded{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Distributed sync insertation timeout",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
}
],
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A"
}
],
"title": "Distributed",
"type": "row"
},
{
"collapsed": true,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 139
},
"id": 88,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of active tasks in BackgroundProcessingPool (merges, mutations, or replication queue bookkeeping)",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 28
},
"hiddenSeries": false,
"id": 95,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_BackgroundMergesAndMutationsPoolTask{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "BackgroundPool task",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of active tasks in BackgroundFetchesPool",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 28
},
"hiddenSeries": false,
"id": 96,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_BackgroundFetchesPoolTask{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "BackgroundFetchesPool task",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of active tasks in BackgroundProcessingPool for moves",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 36
},
"hiddenSeries": false,
"id": 97,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_BackgroundMovePoolTask{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "BackgroundMovePool task",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of active tasks in BackgroundSchedulePool. This pool is used for periodic ReplicatedMergeTree tasks, like cleaning old data parts, altering data parts, replica re-initialization, etc.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 36
},
"hiddenSeries": false,
"id": 98,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_BackgroundSchedulePoolTask{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "BackgroundSchedulePool task",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of active tasks in BackgroundBufferFlushSchedulePool. This pool is used for periodic Buffer flushes",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 44
},
"hiddenSeries": false,
"id": 99,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_BackgroundBufferFlushSchedulePoolTask{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "BackgroundBufferFlushSchedulePool task",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of active tasks in BackgroundDistributedSchedulePool. This pool is used for distributed sends that is done in background.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 44
},
"hiddenSeries": false,
"id": 100,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_BackgroundDistributedSchedulePoolTask{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "BackgroundDistributedSchedulePool task",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of active tasks in BackgroundProcessingPool for message streaming",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 52
},
"hiddenSeries": false,
"id": 101,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_BackgroundMessageBrokerSchedulePoolTask{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "BackgroundMessageBrokerSchedulePool task",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
}
],
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A"
}
],
"title": "Background pool",
"type": "row"
},
{
"collapsed": true,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 140
},
"id": 28,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 29
},
"hiddenSeries": false,
"id": 30,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperInit{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ZooKeeperInit{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Init",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 29
},
"hiddenSeries": false,
"id": 31,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperTransactions{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ZooKeeperTransactions{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Transactions",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 37
},
"hiddenSeries": false,
"id": 32,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperList{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ZooKeeperList{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "List",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 37
},
"hiddenSeries": false,
"id": 33,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperCreate{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ZooKeeperCreate{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Create",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 45
},
"hiddenSeries": false,
"id": 34,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperRemove{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ZooKeeperRemove{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Remove",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 45
},
"hiddenSeries": false,
"id": 35,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperExists{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ZooKeeperExists{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Exists",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 53
},
"hiddenSeries": false,
"id": 36,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperGet{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ZooKeeperGet{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Get",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 53
},
"hiddenSeries": false,
"id": 37,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperSet{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ZooKeeperSet{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Set",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 61
},
"hiddenSeries": false,
"id": 38,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperMulti{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ZooKeeperMulti{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Multi",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 61
},
"hiddenSeries": false,
"id": 39,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperCheck{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ZooKeeperCheck{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Check",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 69
},
"hiddenSeries": false,
"id": 40,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperClose{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ZooKeeperClose{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Close",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 69
},
"hiddenSeries": false,
"id": 41,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperWatchResponse{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ZooKeeperWatchResponse{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Watch response",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "reqps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 77
},
"hiddenSeries": false,
"id": 42,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperUserExceptions{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ZooKeeperUserExceptions{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "User Exceptions",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 77
},
"hiddenSeries": false,
"id": 43,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperHardwareExceptions{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ZooKeeperHardwareExceptions{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Hardware Exceptions",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 85
},
"hiddenSeries": false,
"id": 46,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperBytesReceived{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ZooKeeperBytesReceived{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Bytes received",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "binBps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 85
},
"hiddenSeries": false,
"id": 45,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_ZooKeeperBytesSent{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_ZooKeeperBytesSent{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Bytes sent",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "binBps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of sessions (connections) to ZooKeeper. Should be no more than one, because using more than one connection to ZooKeeper may lead to bugs due to lack of linearizability (stale reads) that ZooKeeper consistency model allows.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 93
},
"hiddenSeries": false,
"id": 48,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_ZooKeeperSession{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Sessions",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of ephemeral nodes hold in ZooKeeper.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 93
},
"hiddenSeries": false,
"id": 47,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_EphemeralNode{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Ephemeral Node",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of watches (event subscriptions) in ZooKeeper.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 101
},
"hiddenSeries": false,
"id": 50,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_ZooKeeperWatch{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Watches",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of requests to ZooKeeper in fly.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 101
},
"hiddenSeries": false,
"id": 49,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "ClickHouseMetrics_ZooKeeperRequest{namespace=~\"$namespace\", pod=~\"$pod\"}",
"interval": "",
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Requests",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "short",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"description": "Number of times an error happened while trying to remove ephemeral node. This is not an issue, because our implementation of ZooKeeper library guarantee that the session will expire and the node will be removed.",
"fieldConfig": {
"defaults": {},
"overrides": []
},
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 109
},
"hiddenSeries": false,
"id": 70,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"maxDataPoints": 200,
"nullPointMode": "null",
"options": {
"alertThreshold": true
},
"percentage": false,
"pluginVersion": "7.5.2",
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "max_over_time( (irate(ClickHouseProfileEvents_CannotRemoveEphemeralNode{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[2m]))[$__rate_interval:15s] ) * $peaks",
"interval": "",
"legendFormat": "peaks - {{pod}}",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"exemplar": false,
"expr": "rate(ClickHouseProfileEvents_CannotRemoveEphemeralNode{instance=~\"$instance\", namespace=~\"$namespace\", pod=~\"$pod\"}[$__rate_interval]) * $trends",
"hide": false,
"interval": "",
"legendFormat": "trend - {{pod}}",
"refId": "C"
}
],
"thresholds": [],
"timeRegions": [],
"title": "Remove ephemeral node failed",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"transparent": true,
"type": "graph",
"xaxis": {
"mode": "time",
"show": true,
"values": []
},
"yaxes": [
{
"$$hashKey": "object:176",
"format": "cps",
"label": "",
"logBase": 1,
"show": true
},
{
"$$hashKey": "object:177",
"format": "short",
"logBase": 1,
"show": true
}
],
"yaxis": {
"align": false
}
}
],
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"refId": "A"
}
],
"title": "Zookeeper",
"type": "row"
}
],
"refresh": "30s",
"schemaVersion": 37,
"style": "dark",
"tags": [
"ClickHouse",
"DB"
],
"templating": {
"list": [
{
"current": {
"selected": false,
"text": "Prometheus",
"value": "Prometheus"
},
"hide": 0,
"includeAll": false,
"label": "DataSource",
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"queryValue": "",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"type": "datasource"
},
{
"current": {
"selected": true,
"text": [
"All"
],
"value": [
"$__all"
]
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"definition": "label_values(ClickHouseAsyncMetrics_Uptime, instance)",
"hide": 0,
"includeAll": true,
"label": "Node",
"multi": true,
"name": "instance",
"options": [],
"query": {
"query": "label_values(ClickHouseAsyncMetrics_Uptime, instance)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 5,
"tagValuesQuery": "",
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"current": {
"selected": true,
"text": "Yes",
"value": "1"
},
"hide": 0,
"includeAll": false,
"label": "Show trends",
"multi": false,
"name": "trends",
"options": [
{
"selected": true,
"text": "Yes",
"value": "1"
},
{
"selected": false,
"text": "No",
"value": "null"
}
],
"query": "Yes : 1, No : null",
"queryValue": "",
"skipUrlSync": false,
"type": "custom"
},
{
"current": {
"selected": false,
"text": "No",
"value": "null"
},
"description": "Be aware of the points limit per timeseries for Grafana. This option may not work for a time range > 24h",
"hide": 0,
"includeAll": false,
"label": "Show peaks",
"multi": false,
"name": "peaks",
"options": [
{
"selected": false,
"text": "Yes",
"value": "1"
},
{
"selected": true,
"text": "No",
"value": "null"
}
],
"query": "Yes : 1, No : null",
"queryValue": "",
"skipUrlSync": false,
"type": "custom"
},
{
"current": {
"isNone": true,
"selected": false,
"text": "None",
"value": ""
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"definition": "label_values(up{job=\"kube-state-metrics\"}, cluster)",
"hide": 2,
"includeAll": false,
"multi": false,
"name": "cluster",
"options": [],
"query": {
"query": "label_values(up{job=\"kube-state-metrics\"}, cluster)",
"refId": "StandardVariableQuery"
},
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
},
{
"current": {
"selected": false,
"text": "All",
"value": "$__all"
},
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"definition": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)",
"hide": 0,
"includeAll": true,
"multi": false,
"name": "namespace",
"options": [],
"query": {
"query": "label_values(kube_namespace_status_phase{job=\"kube-state-metrics\", cluster=\"$cluster\"}, namespace)",
"refId": "StandardVariableQuery"
},
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
},
{
"current": {
"selected": false,
"text": "All",
"value": "$__all"
},
"definition": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, pod)",
"hide": 0,
"includeAll": true,
"multi": false,
"name": "pod",
"options": [],
"query": {
"query": "label_values(kube_pod_info{job=\"kube-state-metrics\", cluster=\"$cluster\", namespace=\"$namespace\"}, pod)",
"refId": "StandardVariableQuery"
},
"refresh": 2,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
}
]
},
"time": {
"from": "now-3h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "ClickHouse (by Namespace)",
"uid": "Afaemoh0",
"version": 4,
"weekStart": ""
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment