mirror of
https://github.com/hanzoai/traces.git
synced 2026-11-03 04:05:45 +00:00
Javiandzeekay
92fc1812cf
* change: remove all traces of ingesters from the dashboards * changelog * fix live store matcher
5973 lines
129 KiB
JSON
5973 lines
129 KiB
JSON
{
|
|
"annotations": {
|
|
"list": [
|
|
{
|
|
"builtIn": 1,
|
|
"datasource": {
|
|
"type": "datasource",
|
|
"uid": "grafana"
|
|
},
|
|
"enable": true,
|
|
"hide": true,
|
|
"iconColor": "rgba(0, 211, 255, 1)",
|
|
"name": "Annotations & Alerts",
|
|
"target": {
|
|
"limit": 100,
|
|
"matchAny": false,
|
|
"tags": [
|
|
|
|
],
|
|
"type": "dashboard"
|
|
},
|
|
"type": "dashboard"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$logsds"
|
|
},
|
|
"enable": true,
|
|
"expr": "{cluster=\"$cluster\", diff_namespace=\"$namespace\", container=\"kube-diff-logger\"}",
|
|
"hide": true,
|
|
"iconColor": "rgba(255, 96, 96, 1)",
|
|
"name": "diffs",
|
|
"showIn": 0,
|
|
"target": {
|
|
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"editable": true,
|
|
"fiscalYearStartMonth": 0,
|
|
"graphTooltip": 1,
|
|
"id": 99,
|
|
"links": [
|
|
{
|
|
"asDropdown": true,
|
|
"icon": "external link",
|
|
"includeVars": true,
|
|
"keepTime": true,
|
|
"tags": [
|
|
"tempo"
|
|
],
|
|
"targetBlank": false,
|
|
"title": "Tempo Dashboards",
|
|
"type": "dashboards"
|
|
}
|
|
],
|
|
"panels": [
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 0
|
|
},
|
|
"id": 38,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 3,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"id": 24,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"hideZeros": false,
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"pluginVersion": "11.5.2",
|
|
"targets": [
|
|
{
|
|
"expr": "rate(go_gc_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval])",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "gcs",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "decbytes"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 3,
|
|
"x": 3,
|
|
"y": 1
|
|
},
|
|
"id": 25,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"hideZeros": false,
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"pluginVersion": "11.5.2",
|
|
"targets": [
|
|
{
|
|
"expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "go heap",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 3,
|
|
"x": 6,
|
|
"y": 1
|
|
},
|
|
"id": 23,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"hideZeros": false,
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "11.5.2",
|
|
"targets": [
|
|
{
|
|
"expr": "go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}",
|
|
"legendFormat": "{{pod}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "goroutines",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "continuous-BlYlRd"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 3,
|
|
"x": 9,
|
|
"y": 1
|
|
},
|
|
"id": 42,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"hideZeros": false,
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"pluginVersion": "11.5.2",
|
|
"targets": [
|
|
{
|
|
"expr": "rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\", container!=\"POD\"}[$__rate_interval])",
|
|
"interval": "",
|
|
"intervalFactor": 5,
|
|
"legendFormat": "{{pod}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "cpu",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "decbytes"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 3,
|
|
"x": 12,
|
|
"y": 1
|
|
},
|
|
"id": 43,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"hideZeros": false,
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"pluginVersion": "11.5.2",
|
|
"targets": [
|
|
{
|
|
"expr": "container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\", container!=\"POD\"}",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "working set",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "decbytes"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 3,
|
|
"x": 15,
|
|
"y": 1
|
|
},
|
|
"id": 44,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"hideZeros": false,
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"pluginVersion": "11.5.2",
|
|
"targets": [
|
|
{
|
|
"expr": "rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "rx-{{pod}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\"}[$__rate_interval])",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "tx-{{pod}}",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "tx/rx",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "decbytes"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 3,
|
|
"x": 18,
|
|
"y": 1
|
|
},
|
|
"id": 45,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"hideZeros": false,
|
|
"mode": "multi",
|
|
"sort": "asc"
|
|
}
|
|
},
|
|
"pluginVersion": "11.5.2",
|
|
"targets": [
|
|
{
|
|
"expr": "kubelet_volume_stats_available_bytes{cluster=\"$cluster\", namespace=\"$namespace\", persistentvolumeclaim=~\"$component.*\"}",
|
|
"legendFormat": "{{persistentvolumeclaim}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "data volume free",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 3,
|
|
"x": 21,
|
|
"y": 1
|
|
},
|
|
"id": 46,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"hideZeros": false,
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"pluginVersion": "11.5.2",
|
|
"targets": [
|
|
{
|
|
"expr": "rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", app=~\"$component.*\"}[$__rate_interval])",
|
|
"interval": "",
|
|
"legendFormat": "{{exported_pod}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "bad words",
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"title": "General",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 1
|
|
},
|
|
"id": 49,
|
|
"panels": [
|
|
|
|
],
|
|
"title": "Maintenance",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "percentunit"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 3,
|
|
"x": 0,
|
|
"y": 2
|
|
},
|
|
"id": 33,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"hideZeros": false,
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"pluginVersion": "11.5.2",
|
|
"targets": [
|
|
{
|
|
"expr": "tempodb_work_queue_length{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"} / tempodb_work_queue_max{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}",
|
|
"legendFormat": "{{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "%age total work queue",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 3,
|
|
"x": 3,
|
|
"y": 2
|
|
},
|
|
"id": 32,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"hideZeros": false,
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"pluginVersion": "11.5.2",
|
|
"targets": [
|
|
{
|
|
"expr": "sum(increase(tempodb_compaction_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval])) by (job)",
|
|
"legendFormat": "compaction_err",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"expr": "sum(increase(tempodb_retention_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval])) by (job)",
|
|
"legendFormat": "retention_err",
|
|
"refId": "C"
|
|
},
|
|
{
|
|
"expr": "sum(increase(tempodb_blocklist_poll_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval])) by (job)",
|
|
"legendFormat": "blocklist_err",
|
|
"refId": "D"
|
|
}
|
|
],
|
|
"title": "maintenance errors",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": false,
|
|
"type": "prometheus",
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "points",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 3,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "always",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 3,
|
|
"x": 6,
|
|
"y": 2
|
|
},
|
|
"id": 35,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"hideZeros": false,
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "11.5.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "mimir-ops-03"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.99, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval])) by (le)) < ($latency_metrics * +Inf)",
|
|
"interval": "",
|
|
"legendFormat": ".99",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "mimir-ops-03"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.9, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval])) by (le)) < ($latency_metrics * +Inf)",
|
|
"legendFormat": ".9",
|
|
"range": true,
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "mimir-ops-03"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.5, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval])) by (le)) < ($latency_metrics * +Inf)",
|
|
"interval": "",
|
|
"legendFormat": ".5",
|
|
"range": true,
|
|
"refId": "C"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "mimir-ops-03"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.99, sum(rate(tempodb_blocklist_poll_duration_seconds{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval])) by (le)) < ($latency_metrics * -Inf)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": ".99",
|
|
"range": true,
|
|
"refId": "D"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "mimir-ops-03"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.9, sum(rate(tempodb_blocklist_poll_duration_seconds{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval])) by (le)) < ($latency_metrics * -Inf)",
|
|
"hide": false,
|
|
"legendFormat": ".9",
|
|
"range": true,
|
|
"refId": "E"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "mimir-ops-03"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.5, sum(rate(tempodb_blocklist_poll_duration_seconds{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval])) by (le)) < ($latency_metrics * -Inf)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": ".5",
|
|
"range": true,
|
|
"refId": "F"
|
|
}
|
|
],
|
|
"title": "Blocklist Poll Duration",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "normal"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 3,
|
|
"x": 9,
|
|
"y": 2
|
|
},
|
|
"id": 47,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"hideZeros": false,
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "11.5.2",
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "avg(tempodb_blocklist_length{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}) by (tenant)",
|
|
"instant": false,
|
|
"interval": "",
|
|
"legendFormat": "{{tenant}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Blocklist Length",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": false,
|
|
"type": "prometheus",
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "points",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 4,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "always",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 3,
|
|
"x": 12,
|
|
"y": 2
|
|
},
|
|
"id": 51,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"hideZeros": false,
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "11.5.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "mimir-ops-03"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.99, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le)) < ($latency_metrics * +Inf)",
|
|
"interval": "",
|
|
"legendFormat": ".99",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "mimir-ops-03"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.9, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le)) < ($latency_metrics * +Inf)",
|
|
"interval": "",
|
|
"legendFormat": ".9",
|
|
"range": true,
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "mimir-ops-03"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.5, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le)) < ($latency_metrics * +Inf)",
|
|
"interval": "",
|
|
"legendFormat": ".5",
|
|
"range": true,
|
|
"refId": "C"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "mimir-ops-03"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.99, sum(rate(tempodb_retention_duration_seconds{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval]))) < ($latency_metrics * -Inf)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": ".99",
|
|
"range": true,
|
|
"refId": "D"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "mimir-ops-03"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.9, sum(rate(tempodb_retention_duration_seconds{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval]))) < ($latency_metrics * -Inf)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": ".9",
|
|
"range": true,
|
|
"refId": "E"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "mimir-ops-03"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.5, sum(rate(tempodb_retention_duration_seconds{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) ) < ($latency_metrics * -Inf)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": ".5",
|
|
"range": true,
|
|
"refId": "F"
|
|
}
|
|
],
|
|
"title": "retention duration",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 3,
|
|
"x": 15,
|
|
"y": 2
|
|
},
|
|
"id": 53,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"hideZeros": false,
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"pluginVersion": "11.5.2",
|
|
"targets": [
|
|
{
|
|
"expr": "sum(increase(tempodb_retention_deleted_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval]))",
|
|
"interval": "",
|
|
"legendFormat": "deleted",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "sum(increase(tempodb_retention_marked_for_deletion_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval]))",
|
|
"interval": "",
|
|
"legendFormat": "marked_for_deletion",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "retention",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 3,
|
|
"x": 18,
|
|
"y": 2
|
|
},
|
|
"id": 70,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"hideZeros": false,
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "11.5.2",
|
|
"targets": [
|
|
{
|
|
"expr": "increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]) > 0",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Container Restarts",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 7
|
|
},
|
|
"id": 21,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 4,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"id": 34,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(tempo_request_duration_seconds_count{route=~\".*traces_traceid\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\"}[$__rate_interval])) by (status_code)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{status_code}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Queries/Sec (cortex-gw)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 4,
|
|
"y": 8
|
|
},
|
|
"id": 78,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*traces_traceid\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".99",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*traces_traceid\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".9",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*traces_traceid\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".5",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"title": "Query Latency (Gateway)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 10,
|
|
"y": 8
|
|
},
|
|
"id": 97,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "sum(rate(tempo_request_duration_seconds_count{route=~\".*api_search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\"}[$__rate_interval])) by (status_code)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{status_code}}",
|
|
"refId": "A",
|
|
"stepMode": "min"
|
|
}
|
|
],
|
|
"title": "Search Queries/Sec (cortex-gw)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 8
|
|
},
|
|
"id": 93,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".99",
|
|
"refId": "A",
|
|
"stepMode": "min"
|
|
},
|
|
{
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": ".9",
|
|
"refId": "B",
|
|
"stepMode": "min"
|
|
},
|
|
{
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": ".5",
|
|
"refId": "C",
|
|
"stepMode": "min"
|
|
}
|
|
],
|
|
"title": "Search Query Latency (Gateway)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 4,
|
|
"x": 0,
|
|
"y": 13
|
|
},
|
|
"id": 90,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*traces_traceid\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{status_code}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Queries/Sec (Query Frontend)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 4,
|
|
"y": 13
|
|
},
|
|
"id": 17,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*traces_traceid\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".99",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*traces_traceid\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".9",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*traces_traceid\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".5",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"title": "Query Latency (Query Frontend)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 10,
|
|
"y": 13
|
|
},
|
|
"id": 98,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_search.*\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{status_code}}",
|
|
"refId": "A",
|
|
"stepMode": "min"
|
|
}
|
|
],
|
|
"title": "Search Queries/Sec (Query Frontend)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 13
|
|
},
|
|
"id": 94,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".99",
|
|
"refId": "A",
|
|
"stepMode": "min"
|
|
},
|
|
{
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".9",
|
|
"refId": "B",
|
|
"stepMode": "min"
|
|
},
|
|
{
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".5",
|
|
"refId": "C",
|
|
"stepMode": "min"
|
|
}
|
|
],
|
|
"title": "Search Query Latency (Query Frontend)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 4,
|
|
"x": 0,
|
|
"y": 24
|
|
},
|
|
"id": 91,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"querier_.*traces_traceid\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\"}[$__rate_interval])) by (status_code)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{status_code}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Queries/Sec (Querier)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 4,
|
|
"y": 24
|
|
},
|
|
"id": 89,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*traces_traceid\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".99",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*traces_traceid\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".9",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*traces_traceid\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".5",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"title": "Query Latency (Querier)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 10,
|
|
"y": 24
|
|
},
|
|
"id": 99,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"querier_.*api_search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\"}[$__rate_interval])) by (status_code)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{status_code}}",
|
|
"refId": "A",
|
|
"stepMode": "min"
|
|
}
|
|
],
|
|
"title": "Search Queries/Sec (Querier)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 24
|
|
},
|
|
"id": 95,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".99",
|
|
"refId": "A",
|
|
"stepMode": "min"
|
|
},
|
|
{
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".9",
|
|
"refId": "B",
|
|
"stepMode": "min"
|
|
},
|
|
{
|
|
"exemplar": true,
|
|
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".5",
|
|
"refId": "C",
|
|
"stepMode": "min"
|
|
}
|
|
],
|
|
"title": "Search Query Latency (Querier)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 4,
|
|
"x": 0,
|
|
"y": 34
|
|
},
|
|
"id": 1001,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(tempo_request_duration_seconds_count{route=\"/tempopb.Querier/FindTraceByID\", cluster=\"$cluster\", namespace=\"$namespace\", container=\"live-store\"}[$__rate_interval])) by (status_code)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{status_code}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Queries/Sec (Live Store)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 4,
|
|
"y": 34
|
|
},
|
|
"id": 1002,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"live-store\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".99",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"live-store\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".9",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"live-store\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".5",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"title": "Query Latency (Live Store)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 10,
|
|
"y": 34
|
|
},
|
|
"id": 1003,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"/tempopb.Querier/Search.*\", cluster=\"$cluster\", namespace=\"$namespace\", container=\"live-store\"}[$__rate_interval])) by (status_code)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": "{{status_code}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Search Queries/Sec (Live Store)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 8,
|
|
"x": 16,
|
|
"y": 34
|
|
},
|
|
"id": 1004,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"live-store\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".99",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"live-store\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".9",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", container=\"live-store\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".5",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"title": "Search Query Latency (Live Store)",
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"title": "Read",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 8
|
|
},
|
|
"id": 19,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "normal"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 3,
|
|
"x": 0,
|
|
"y": 20
|
|
},
|
|
"id": 10,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d452322apre",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "P666011C0B63BDCA4"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum by(cluster) (rate(tempo_distributor_spans_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
|
|
"interval": "",
|
|
"legendFormat": "{{pod}}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Spans Received",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 6,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "always",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 3,
|
|
"x": 18,
|
|
"y": 20
|
|
},
|
|
"id": 114,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
"lastNotNull"
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d452322apre",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "cortex-ops-01"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(tempo_distributor_queue_length{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\".*/distributor\"}) by (name)",
|
|
"interval": "",
|
|
"legendFormat": "{{name}}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Distributor Queue Length",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": false,
|
|
"type": "prometheus",
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 2,
|
|
"x": 21,
|
|
"y": 20
|
|
},
|
|
"id": 115,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d452322apre",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "mimir-ops-03"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(tempo_distributor_metrics_generator_pushes_failures_total{namespace=~\"$namespace\", cluster=~\"$cluster\"}[1m])) by (namespace, cluster)",
|
|
"interval": "",
|
|
"legendFormat": "{{cluster}} {{namespace}}",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Pushes Failing (Distributor)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 10,
|
|
"w": 7,
|
|
"x": 0,
|
|
"y": 25
|
|
},
|
|
"id": 71,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d452322apre",
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\"(opentelemetry_proto_collector_trace_v1_traceservice_export|otlp_v1_traces)\"}[$__rate_interval])) by (status_code)",
|
|
"interval": "",
|
|
"legendFormat": "{{status_code}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Pushes/sec (gateway)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 10,
|
|
"w": 6,
|
|
"x": 7,
|
|
"y": 25
|
|
},
|
|
"id": 72,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d452322apre",
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(tempo_receiver_accepted_spans{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
|
|
"interval": "",
|
|
"legendFormat": "accepted",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "sum(rate(tempo_receiver_refused_spans{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
|
|
"interval": "",
|
|
"legendFormat": "refused",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "OTEL Distributor Spans/second",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": false,
|
|
"type": "prometheus",
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 7,
|
|
"x": 13,
|
|
"y": 25
|
|
},
|
|
"id": 79,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d452322apre",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "mimir-ops-03"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\"(opentelemetry_proto_collector_trace_v1_traceservice_export|otlp_v1_traces)\"}[$__rate_interval])) by (le)) > ($latency_metrics * -Inf)",
|
|
"interval": "",
|
|
"legendFormat": ".99",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "mimir-ops-03"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\"(opentelemetry_proto_collector_trace_v1_traceservice_export|otlp_v1_traces)\"}[$__rate_interval])) by (le)) > ($latency_metrics * -Inf)",
|
|
"interval": "",
|
|
"legendFormat": ".9",
|
|
"range": true,
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "mimir-ops-03"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\"(opentelemetry_proto_collector_trace_v1_traceservice_export|otlp_v1_traces)\"}[$__rate_interval])) by (le)) > ($latency_metrics * -Inf)",
|
|
"interval": "",
|
|
"legendFormat": ".5",
|
|
"range": true,
|
|
"refId": "C"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "mimir-ops-03"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\"(opentelemetry_proto_collector_trace_v1_traceservice_export|otlp_v1_traces)\"}[$__rate_interval]))) < ($latency_metrics * -Inf)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": ".99",
|
|
"range": true,
|
|
"refId": "D"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "mimir-ops-03"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\"(opentelemetry_proto_collector_trace_v1_traceservice_export|otlp_v1_traces)\"}[$__rate_interval]))) < ($latency_metrics * -Inf)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": ".9",
|
|
"range": true,
|
|
"refId": "E"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "mimir-ops-03"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/cortex-gw(-internal)?\", route=~\"(opentelemetry_proto_collector_trace_v1_traceservice_export|otlp_v1_traces)\"}[$__rate_interval]))) < ($latency_metrics * -Inf)",
|
|
"hide": false,
|
|
"interval": "",
|
|
"legendFormat": ".5",
|
|
"range": true,
|
|
"refId": "F"
|
|
}
|
|
],
|
|
"title": "Push Latency (Gateway)",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "none"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 6,
|
|
"x": 7,
|
|
"y": 35
|
|
},
|
|
"id": 109,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d452322apre",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "cortex-ops-01"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(tempo_metrics_generator_spans_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
|
|
"interval": "",
|
|
"legendFormat": "accepted",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "cortex-ops-01"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(tempo_metrics_generator_spans_discarded_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (reason)",
|
|
"interval": "",
|
|
"legendFormat": "refused {{reason}}",
|
|
"range": true,
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Generator Spans/second",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"default": false,
|
|
"type": "prometheus",
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 5,
|
|
"w": 7,
|
|
"x": 13,
|
|
"y": 35
|
|
},
|
|
"id": 108,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d452322apre",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "cortex-ops-01"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=~\"/tempopb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".99",
|
|
"range": true,
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "cortex-ops-01"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=~\"/tempopb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".9",
|
|
"range": true,
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "cortex-ops-01"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/metrics-generator\", route=~\"/tempopb.MetricsGenerator/PushSpans\"}[$__rate_interval])) by (le))",
|
|
"interval": "",
|
|
"legendFormat": ".5",
|
|
"range": true,
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"title": "Push Latency (Generator)",
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"title": "Write",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 9
|
|
},
|
|
"id": 74,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"links": [
|
|
|
|
],
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 26
|
|
},
|
|
"id": 75,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(tempo_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval])) by (status_code, method)",
|
|
"interval": "",
|
|
"legendFormat": "{{status_code}}-{{method}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Requests/Second",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"links": [
|
|
|
|
],
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 26
|
|
},
|
|
"id": 76,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval])) by (method, le))",
|
|
"interval": "",
|
|
"legendFormat": ".99-{{method}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(.9, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval])) by (method, le))",
|
|
"interval": "",
|
|
"legendFormat": ".9-{{method}}",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(.5, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval])) by (method, le))",
|
|
"interval": "",
|
|
"legendFormat": ".5-{{method}}",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"title": "Latency By Operation",
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"title": "Memcached",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 10
|
|
},
|
|
"id": 28,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"links": [
|
|
|
|
],
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 27
|
|
},
|
|
"id": 31,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(tempodb_backend_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval])) by (status_code, operation)",
|
|
"interval": "",
|
|
"legendFormat": "{{status_code}}-{{operation}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Requests/Second",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"links": [
|
|
|
|
],
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 27
|
|
},
|
|
"id": 30,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval])) by (operation, le))",
|
|
"legendFormat": ".99-{{operation}}",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(.9, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval])) by (operation, le))",
|
|
"legendFormat": ".9-{{operation}}",
|
|
"refId": "B"
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(.5, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}[$__rate_interval])) by (operation, le))",
|
|
"legendFormat": ".5-{{operation}}",
|
|
"refId": "C"
|
|
}
|
|
],
|
|
"title": "Latency By Operation",
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"title": "Backend",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 11
|
|
},
|
|
"id": 56,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"links": [
|
|
|
|
],
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 6,
|
|
"x": 0,
|
|
"y": 28
|
|
},
|
|
"id": 58,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "P666011C0B63BDCA4"
|
|
},
|
|
"expr": "gauge_memberlist_health_score{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}",
|
|
"interval": "",
|
|
"legendFormat": "{{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Gossip Health",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"links": [
|
|
|
|
],
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 6,
|
|
"x": 6,
|
|
"y": 28
|
|
},
|
|
"id": 62,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"expr": "tempo_memberlist_client_cluster_node_health_score{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"}",
|
|
"interval": "",
|
|
"legendFormat": "{{instance}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Node Health",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"links": [
|
|
|
|
],
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 6,
|
|
"x": 12,
|
|
"y": 28
|
|
},
|
|
"id": 63,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"expr": "min(tempo_memberlist_client_cluster_members_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"})",
|
|
"interval": "",
|
|
"legendFormat": "min",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"expr": "max(tempo_memberlist_client_cluster_members_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"})",
|
|
"interval": "",
|
|
"legendFormat": "max",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "Member Count",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"links": [
|
|
|
|
],
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 8,
|
|
"w": 6,
|
|
"x": 18,
|
|
"y": 28
|
|
},
|
|
"id": 60,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"mode": "multi",
|
|
"sort": "desc"
|
|
}
|
|
},
|
|
"pluginVersion": "9.0.0-d373beebpre",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "P666011C0B63BDCA4"
|
|
},
|
|
"expr": "min(tempo_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"})",
|
|
"interval": "",
|
|
"legendFormat": "min",
|
|
"refId": "A"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "P666011C0B63BDCA4"
|
|
},
|
|
"expr": "max(tempo_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component.*\"})",
|
|
"interval": "",
|
|
"legendFormat": "max",
|
|
"refId": "B"
|
|
}
|
|
],
|
|
"title": "KV Store Count",
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"title": "Ring/Memberlist",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": true,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 12
|
|
},
|
|
"id": 69,
|
|
"panels": [
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 0,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "auto",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "percentunit"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 0,
|
|
"y": 24
|
|
},
|
|
"id": 77,
|
|
"options": {
|
|
"graph": {
|
|
|
|
},
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "7.5.0-14141pre",
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(tempo_vulture_trace_error_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
|
|
"interval": "",
|
|
"legendFormat": "{{error}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Vulture Query Errors",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "thresholds"
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"max": 1,
|
|
"min": 0,
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green"
|
|
}
|
|
]
|
|
},
|
|
"unit": "percentunit"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 9,
|
|
"w": 12,
|
|
"x": 12,
|
|
"y": 24
|
|
},
|
|
"id": 67,
|
|
"options": {
|
|
"displayMode": "gradient",
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"maxVizHeight": 300,
|
|
"minVizHeight": 10,
|
|
"minVizWidth": 0,
|
|
"namePlacement": "auto",
|
|
"orientation": "auto",
|
|
"reduceOptions": {
|
|
"calcs": [
|
|
"mean"
|
|
],
|
|
"fields": "",
|
|
"values": false
|
|
},
|
|
"showUnfilled": true,
|
|
"sizing": "auto",
|
|
"text": {
|
|
|
|
},
|
|
"valueMode": "color"
|
|
},
|
|
"pluginVersion": "11.3.0-75324",
|
|
"targets": [
|
|
{
|
|
"expr": "sum(rate(tempo_vulture_trace_error_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
|
|
"interval": "",
|
|
"legendFormat": "{{secondsago}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Average Vulture Query Errors",
|
|
"type": "bargauge"
|
|
}
|
|
],
|
|
"title": "Vulture",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"collapsed": false,
|
|
"gridPos": {
|
|
"h": 1,
|
|
"w": 24,
|
|
"x": 0,
|
|
"y": 13
|
|
},
|
|
"id": 81,
|
|
"panels": [
|
|
|
|
],
|
|
"title": "Compactor",
|
|
"type": "row"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 4,
|
|
"x": 0,
|
|
"y": 14
|
|
},
|
|
"id": 83,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"hideZeros": false,
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "11.5.2",
|
|
"targets": [
|
|
{
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(tempodb_compaction_objects_combined_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (level)",
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Objects Combined / s",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"uid": "$ds"
|
|
},
|
|
"description": "",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "normal"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
},
|
|
{
|
|
"color": "red",
|
|
"value": 80
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 4,
|
|
"x": 4,
|
|
"y": 14
|
|
},
|
|
"id": 85,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"hideZeros": false,
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "11.5.2",
|
|
"targets": [
|
|
{
|
|
"editorMode": "code",
|
|
"expr": "sum(rate(tempodb_compaction_objects_written_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (level)",
|
|
"interval": "",
|
|
"legendFormat": "",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Objects Written / s",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 4,
|
|
"x": 8,
|
|
"y": 14
|
|
},
|
|
"id": 88,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"hideZeros": false,
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "11.5.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "P666011C0B63BDCA4"
|
|
},
|
|
"disableTextWrap": false,
|
|
"editorMode": "builder",
|
|
"expr": "sum by(level) (rate(tempodb_compaction_bytes_written_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))",
|
|
"fullMetaSearch": false,
|
|
"includeNullMetadata": true,
|
|
"interval": "",
|
|
"legendFormat": "__auto",
|
|
"range": true,
|
|
"refId": "A",
|
|
"useBackend": false
|
|
}
|
|
],
|
|
"title": "Bytes Written / s",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds"
|
|
},
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"color": {
|
|
"mode": "palette-classic"
|
|
},
|
|
"custom": {
|
|
"axisBorderShow": false,
|
|
"axisCenteredZero": false,
|
|
"axisColorMode": "text",
|
|
"axisLabel": "",
|
|
"axisPlacement": "auto",
|
|
"barAlignment": 0,
|
|
"barWidthFactor": 0.6,
|
|
"drawStyle": "line",
|
|
"fillOpacity": 10,
|
|
"gradientMode": "none",
|
|
"hideFrom": {
|
|
"legend": false,
|
|
"tooltip": false,
|
|
"viz": false
|
|
},
|
|
"insertNulls": false,
|
|
"lineInterpolation": "linear",
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"scaleDistribution": {
|
|
"type": "linear"
|
|
},
|
|
"showPoints": "never",
|
|
"spanNulls": true,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "normal"
|
|
},
|
|
"thresholdsStyle": {
|
|
"mode": "off"
|
|
}
|
|
},
|
|
"mappings": [
|
|
|
|
],
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
{
|
|
"color": "green",
|
|
"value": null
|
|
}
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"gridPos": {
|
|
"h": 6,
|
|
"w": 4,
|
|
"x": 12,
|
|
"y": 14
|
|
},
|
|
"id": 86,
|
|
"options": {
|
|
"legend": {
|
|
"calcs": [
|
|
|
|
],
|
|
"displayMode": "list",
|
|
"placement": "bottom",
|
|
"showLegend": false
|
|
},
|
|
"tooltip": {
|
|
"hideZeros": false,
|
|
"mode": "multi",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"pluginVersion": "11.5.2",
|
|
"targets": [
|
|
{
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "P666011C0B63BDCA4"
|
|
},
|
|
"editorMode": "code",
|
|
"expr": "sum(increase(tempodb_compaction_blocks_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (level)",
|
|
"interval": "",
|
|
"legendFormat": "__auto",
|
|
"range": true,
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "Blocks Compacted",
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"preload": false,
|
|
"refresh": "",
|
|
"schemaVersion": 40,
|
|
"tags": [
|
|
"tempo"
|
|
],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"current": {
|
|
"text": "default",
|
|
"value": "default"
|
|
},
|
|
"includeAll": false,
|
|
"name": "ds",
|
|
"options": [
|
|
|
|
],
|
|
"query": "prometheus",
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"type": "datasource"
|
|
},
|
|
{
|
|
"current": {
|
|
"text": "loki-ops",
|
|
"value": "loki-ops"
|
|
},
|
|
"includeAll": false,
|
|
"name": "logsds",
|
|
"options": [
|
|
|
|
],
|
|
"query": "loki",
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"type": "datasource"
|
|
},
|
|
{
|
|
"current": {
|
|
"text": "ops-eu-south-0",
|
|
"value": "ops-eu-south-0"
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds"
|
|
},
|
|
"definition": "label_values(tempo_build_info, cluster)",
|
|
"includeAll": false,
|
|
"name": "cluster",
|
|
"options": [
|
|
|
|
],
|
|
"query": {
|
|
"query": "label_values(tempo_build_info, cluster)",
|
|
"refId": "ops-cortex-cluster-Variable-Query"
|
|
},
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"sort": 1,
|
|
"type": "query"
|
|
},
|
|
{
|
|
"current": {
|
|
"text": "tempo-ops-01",
|
|
"value": "tempo-ops-01"
|
|
},
|
|
"datasource": {
|
|
"type": "prometheus",
|
|
"uid": "$ds"
|
|
},
|
|
"definition": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)",
|
|
"includeAll": false,
|
|
"name": "namespace",
|
|
"options": [
|
|
|
|
],
|
|
"query": {
|
|
"query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)",
|
|
"refId": "ops-cortex-namespace-Variable-Query"
|
|
},
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"sort": 1,
|
|
"type": "query"
|
|
},
|
|
{
|
|
"allValue": ".*",
|
|
"current": {
|
|
"text": "All",
|
|
"value": "$__all"
|
|
},
|
|
"includeAll": true,
|
|
"name": "component",
|
|
"options": [
|
|
{
|
|
"selected": false,
|
|
"text": "All",
|
|
"value": "$__all"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "block-builder",
|
|
"value": "block-builder"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "compactor",
|
|
"value": "compactor"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "distributor",
|
|
"value": "distributor"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "metrics-generator",
|
|
"value": "metrics-generator"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "query-frontend",
|
|
"value": "query-frontend"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "querier",
|
|
"value": "querier"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "cortex-gw",
|
|
"value": "cortex-gw"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "cortex-gw-internal",
|
|
"value": "cortex-gw-internal"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "live-store",
|
|
"value": "live-store"
|
|
}
|
|
],
|
|
"query": "block-builder,compactor,distributor,metrics-generator,query-frontend,querier,cortex-gw,cortex-gw-internal,live-store",
|
|
"type": "custom"
|
|
},
|
|
{
|
|
"current": {
|
|
"text": "native",
|
|
"value": "-1"
|
|
},
|
|
"description": "Choose between showing latencies based on low precision classic or high precision native histogram metrics.",
|
|
"includeAll": false,
|
|
"label": "Latency metrics",
|
|
"name": "latency_metrics",
|
|
"options": [
|
|
{
|
|
"selected": true,
|
|
"text": "native",
|
|
"value": "-1"
|
|
},
|
|
{
|
|
"selected": false,
|
|
"text": "classic",
|
|
"value": "1"
|
|
}
|
|
],
|
|
"query": "native : -1,classic : 1",
|
|
"type": "custom"
|
|
}
|
|
]
|
|
},
|
|
"time": {
|
|
"from": "now-6h",
|
|
"to": "now"
|
|
},
|
|
"timepicker": {
|
|
"refresh_intervals": [
|
|
"10s",
|
|
"30s",
|
|
"1m",
|
|
"5m",
|
|
"15m",
|
|
"30m",
|
|
"1h",
|
|
"2h",
|
|
"1d"
|
|
]
|
|
},
|
|
"timezone": "",
|
|
"title": "Tempo Operational",
|
|
"uid": "a6175b9cc7ec20591890117c39580030",
|
|
"version": 2,
|
|
"weekStart": ""
|
|
} |