mirror of
https://github.com/hanzoai/traces.git
synced 2026-11-03 04:05:45 +00:00
61a4ea3612
Add QPS and Latency panels for the external endpoint metric (tempo_querier_external_endpoint_request_duration_seconds). Co-authored-by: Cursor <[email protected]>
1673 lines
38 KiB
JSON
1673 lines
38 KiB
JSON
{
|
|
"annotations": {
|
|
"list": [
|
|
|
|
]
|
|
},
|
|
"editable": true,
|
|
"gnetId": null,
|
|
"graphTooltip": 0,
|
|
"hideControls": false,
|
|
"links": [
|
|
{
|
|
"asDropdown": true,
|
|
"icon": "external link",
|
|
"includeVars": true,
|
|
"keepTime": true,
|
|
"tags": [
|
|
"tempo"
|
|
],
|
|
"targetBlank": false,
|
|
"title": "Tempo Dashboards",
|
|
"type": "dashboards"
|
|
}
|
|
],
|
|
"refresh": "",
|
|
"rows": [
|
|
{
|
|
"collapse": false,
|
|
"height": "250px",
|
|
"panels": [
|
|
{
|
|
"aliasColors": {
|
|
"1xx": "#EAB839",
|
|
"2xx": "#7EB26D",
|
|
"3xx": "#6ED0E0",
|
|
"4xx": "#EF843C",
|
|
"5xx": "#E24D42",
|
|
"OK": "#7EB26D",
|
|
"cancel": "#A9A9A9",
|
|
"error": "#E24D42",
|
|
"success": "#7EB26D"
|
|
},
|
|
"datasource": "$datasource",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 1,
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "ops"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"fill": 10,
|
|
"id": 1,
|
|
"linewidth": 0,
|
|
"links": [
|
|
|
|
],
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"span": 6,
|
|
"stack": true,
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\", route=~\"api_.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z_]+)\"))\n",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"legendFormat": "{{status}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "QPS",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": "$datasource",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 1,
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "ms"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"id": 2,
|
|
"links": [
|
|
|
|
],
|
|
"nullPointMode": "null as zero",
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"span": 6,
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\",route=~\"api_.*\"}[$__rate_interval])) by (le,route)) * 1e3",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} 99th",
|
|
"refId": "A",
|
|
"step": 10
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\",route=~\"api_.*\"}[$__rate_interval])) by (le,route)) * 1e3",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} 50th",
|
|
"refId": "B",
|
|
"step": 10
|
|
},
|
|
{
|
|
"expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\",route=~\"api_.*\"}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/cortex-gw(-internal)?\",route=~\"api_.*\"}[$__rate_interval])) by (route)",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} Average",
|
|
"refId": "C",
|
|
"step": 10
|
|
}
|
|
],
|
|
"title": "Latency",
|
|
"type": "timeseries",
|
|
"yaxes": [
|
|
{
|
|
"format": "ms",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": 0,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"repeat": null,
|
|
"repeatIteration": null,
|
|
"repeatRowId": null,
|
|
"showTitle": true,
|
|
"title": "Gateway",
|
|
"titleSize": "h6"
|
|
},
|
|
{
|
|
"collapse": false,
|
|
"height": "250px",
|
|
"panels": [
|
|
{
|
|
"aliasColors": {
|
|
"1xx": "#EAB839",
|
|
"2xx": "#7EB26D",
|
|
"3xx": "#6ED0E0",
|
|
"4xx": "#EF843C",
|
|
"5xx": "#E24D42",
|
|
"OK": "#7EB26D",
|
|
"cancel": "#A9A9A9",
|
|
"error": "#E24D42",
|
|
"success": "#7EB26D"
|
|
},
|
|
"datasource": "$datasource",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 1,
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "ops"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"fill": 10,
|
|
"id": 3,
|
|
"linewidth": 0,
|
|
"links": [
|
|
|
|
],
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"span": 6,
|
|
"stack": true,
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\", route=~\"api_.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z_]+)\"))\n",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"legendFormat": "{{status}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "QPS",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": "$datasource",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 1,
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "ms"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"id": 4,
|
|
"links": [
|
|
|
|
],
|
|
"nullPointMode": "null as zero",
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"span": 6,
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__rate_interval])) by (le,route)) * 1e3",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} 99th",
|
|
"refId": "A",
|
|
"step": 10
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__rate_interval])) by (le,route)) * 1e3",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} 50th",
|
|
"refId": "B",
|
|
"step": 10
|
|
},
|
|
{
|
|
"expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/query-frontend\",route=~\"api_.*\"}[$__rate_interval])) by (route)",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} Average",
|
|
"refId": "C",
|
|
"step": 10
|
|
}
|
|
],
|
|
"title": "Latency",
|
|
"type": "timeseries",
|
|
"yaxes": [
|
|
{
|
|
"format": "ms",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": 0,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"repeat": null,
|
|
"repeatIteration": null,
|
|
"repeatRowId": null,
|
|
"showTitle": true,
|
|
"title": "Query Frontend",
|
|
"titleSize": "h6"
|
|
},
|
|
{
|
|
"collapse": false,
|
|
"height": "250px",
|
|
"panels": [
|
|
{
|
|
"aliasColors": {
|
|
"1xx": "#EAB839",
|
|
"2xx": "#7EB26D",
|
|
"3xx": "#6ED0E0",
|
|
"4xx": "#EF843C",
|
|
"5xx": "#E24D42",
|
|
"OK": "#7EB26D",
|
|
"cancel": "#A9A9A9",
|
|
"error": "#E24D42",
|
|
"success": "#7EB26D"
|
|
},
|
|
"datasource": "$datasource",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 1,
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "ops"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"fill": 10,
|
|
"id": 5,
|
|
"linewidth": 0,
|
|
"links": [
|
|
|
|
],
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"span": 4,
|
|
"stack": true,
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\", route=~\"querier_api_.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z_]+)\"))\n",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"legendFormat": "{{status}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "QPS",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": "$datasource",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 1,
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "ms"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"id": 6,
|
|
"links": [
|
|
|
|
],
|
|
"nullPointMode": "null as zero",
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"span": 4,
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__rate_interval])) by (le,route)) * 1e3",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} 99th",
|
|
"refId": "A",
|
|
"step": 10
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__rate_interval])) by (le,route)) * 1e3",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} 50th",
|
|
"refId": "B",
|
|
"step": 10
|
|
},
|
|
{
|
|
"expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__rate_interval])) by (route)",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} Average",
|
|
"refId": "C",
|
|
"step": 10
|
|
}
|
|
],
|
|
"title": "Latency",
|
|
"type": "timeseries",
|
|
"yaxes": [
|
|
{
|
|
"format": "ms",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": 0,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"datasource": "$datasource",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 1,
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "ms"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"id": 7,
|
|
"links": [
|
|
|
|
],
|
|
"nullPointMode": "null as zero",
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"span": 4,
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(0.99, sum(rate(tempo_querier_worker_request_executed_total_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__rate_interval])) by (le,route)) * 1e3",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} 99th",
|
|
"refId": "A",
|
|
"step": 10
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.50, sum(rate(tempo_querier_worker_request_executed_total_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__rate_interval])) by (le,route)) * 1e3",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} 50th",
|
|
"refId": "B",
|
|
"step": 10
|
|
},
|
|
{
|
|
"expr": "sum(rate(tempo_querier_worker_request_executed_total_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_querier_worker_request_executed_total_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",route=~\"querier_api_.*\"}[$__rate_interval])) by (route)",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} Average",
|
|
"refId": "C",
|
|
"step": 10
|
|
}
|
|
],
|
|
"title": "Requests Executed",
|
|
"type": "timeseries",
|
|
"yaxes": [
|
|
{
|
|
"format": "ms",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": 0,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"repeat": null,
|
|
"repeatIteration": null,
|
|
"repeatRowId": null,
|
|
"showTitle": true,
|
|
"title": "Querier",
|
|
"titleSize": "h6"
|
|
},
|
|
{
|
|
"collapse": false,
|
|
"height": "250px",
|
|
"panels": [
|
|
{
|
|
"aliasColors": {
|
|
"1xx": "#EAB839",
|
|
"2xx": "#7EB26D",
|
|
"3xx": "#6ED0E0",
|
|
"4xx": "#EF843C",
|
|
"5xx": "#E24D42",
|
|
"OK": "#7EB26D",
|
|
"cancel": "#A9A9A9",
|
|
"error": "#E24D42",
|
|
"success": "#7EB26D"
|
|
},
|
|
"datasource": "$datasource",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 1,
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "ops"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"fill": 10,
|
|
"id": 8,
|
|
"linewidth": 0,
|
|
"links": [
|
|
|
|
],
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"span": 3,
|
|
"stack": true,
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", container=\"live-store\", route=~\"/tempopb.Querier/.*\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z_]+)\"))\n",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"legendFormat": "{{status}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "QPS",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": "$datasource",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 1,
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "ms"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"id": 9,
|
|
"links": [
|
|
|
|
],
|
|
"nullPointMode": "null as zero",
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"span": 3,
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(0.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", container=\"live-store\",route=~\"/tempopb.Querier/.*\"}[$__rate_interval])) by (le,route)) * 1e3",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} 99th",
|
|
"refId": "A",
|
|
"step": 10
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.50, sum(rate(tempo_request_duration_seconds_bucket{cluster=~\"$cluster\", container=\"live-store\",route=~\"/tempopb.Querier/.*\"}[$__rate_interval])) by (le,route)) * 1e3",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} 50th",
|
|
"refId": "B",
|
|
"step": 10
|
|
},
|
|
{
|
|
"expr": "sum(rate(tempo_request_duration_seconds_sum{cluster=~\"$cluster\", container=\"live-store\",route=~\"/tempopb.Querier/.*\"}[$__rate_interval])) by (route) * 1e3 / sum(rate(tempo_request_duration_seconds_count{cluster=~\"$cluster\", container=\"live-store\",route=~\"/tempopb.Querier/.*\"}[$__rate_interval])) by (route)",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} Average",
|
|
"refId": "C",
|
|
"step": 10
|
|
}
|
|
],
|
|
"title": "Latency",
|
|
"type": "timeseries",
|
|
"yaxes": [
|
|
{
|
|
"format": "ms",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": 0,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"datasource": "$datasource",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 1,
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"id": 10,
|
|
"links": [
|
|
|
|
],
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"span": 3,
|
|
"targets": [
|
|
{
|
|
"expr": "tempo_live_store_complete_queue_length{cluster=~\"$cluster\", pod=~\"live-store-zone.*\"}",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"legendFormat": "{{pod}}",
|
|
"legendLink": null
|
|
}
|
|
],
|
|
"title": "Pending Queue Length",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": "$datasource",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 1,
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "ops"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"id": 11,
|
|
"links": [
|
|
|
|
],
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"span": 3,
|
|
"targets": [
|
|
{
|
|
"expr": "sum by(pod) (rate(tempo_live_store_blocks_completed_total{cluster=~\"$cluster\", container=\"live-store\"}[$__rate_interval]))",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"legendFormat": "{{pod}}",
|
|
"legendLink": null
|
|
}
|
|
],
|
|
"title": "Completed Blocks",
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"repeat": null,
|
|
"repeatIteration": null,
|
|
"repeatRowId": null,
|
|
"showTitle": true,
|
|
"title": "Livestore",
|
|
"titleSize": "h6"
|
|
},
|
|
{
|
|
"collapse": false,
|
|
"height": "250px",
|
|
"panels": [
|
|
{
|
|
"datasource": "$datasource",
|
|
"description": "### Kafka lag by partition records\nOverview of the lag by partition in records.\n",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 1,
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "short"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"fill": 10,
|
|
"id": 12,
|
|
"linewidth": 0,
|
|
"links": [
|
|
|
|
],
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"span": 6,
|
|
"stack": true,
|
|
"targets": [
|
|
{
|
|
"expr": "max(tempo_ingest_group_partition_lag{cluster=~\"$cluster\", job=~\"($namespace)/live-store-zone.*\"}) by (partition,group)",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"legendFormat": "{{partition}}",
|
|
"legendLink": null
|
|
}
|
|
],
|
|
"title": "Lag of records by partition",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": "$datasource",
|
|
"description": "### Kafka lag by partition in seconds\nOverview of the lag by partition in seconds.\n",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 1,
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "s"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"fill": 10,
|
|
"id": 13,
|
|
"linewidth": 0,
|
|
"links": [
|
|
|
|
],
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"span": 6,
|
|
"stack": true,
|
|
"targets": [
|
|
{
|
|
"expr": "max(tempo_ingest_group_partition_lag_seconds{cluster=~\"$cluster\", job=~\"($namespace)/live-store-zone.*\"}) by (partition,group)",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"legendFormat": "{{partition}}",
|
|
"legendLink": null
|
|
}
|
|
],
|
|
"title": "Lag by partition (sec)",
|
|
"type": "timeseries"
|
|
}
|
|
],
|
|
"repeat": null,
|
|
"repeatIteration": null,
|
|
"repeatRowId": null,
|
|
"showTitle": true,
|
|
"title": "Livestore Partitions",
|
|
"titleSize": "h6"
|
|
},
|
|
{
|
|
"collapse": false,
|
|
"height": "250px",
|
|
"panels": [
|
|
{
|
|
"aliasColors": {
|
|
"1xx": "#EAB839",
|
|
"2xx": "#7EB26D",
|
|
"3xx": "#6ED0E0",
|
|
"4xx": "#EF843C",
|
|
"5xx": "#E24D42",
|
|
"OK": "#7EB26D",
|
|
"cancel": "#A9A9A9",
|
|
"error": "#E24D42",
|
|
"success": "#7EB26D"
|
|
},
|
|
"datasource": "$datasource",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 1,
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "ops"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"fill": 10,
|
|
"id": 14,
|
|
"linewidth": 0,
|
|
"links": [
|
|
|
|
],
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"span": 6,
|
|
"stack": true,
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z_]+)\"))\n",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"legendFormat": "{{status}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "QPS",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": "$datasource",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 1,
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "ms"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"id": 15,
|
|
"links": [
|
|
|
|
],
|
|
"nullPointMode": "null as zero",
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"span": 6,
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(0.99, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval])) by (le,)) * 1e3",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} 99th",
|
|
"refId": "A",
|
|
"step": 10
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.50, sum(rate(tempo_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval])) by (le,)) * 1e3",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} 50th",
|
|
"refId": "B",
|
|
"step": 10
|
|
},
|
|
{
|
|
"expr": "sum(rate(tempo_memcache_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_memcache_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",method=~\"Memcache.Get|Memcache.GetMulti\"}[$__rate_interval])) by ()",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} Average",
|
|
"refId": "C",
|
|
"step": 10
|
|
}
|
|
],
|
|
"title": "Latency",
|
|
"type": "timeseries",
|
|
"yaxes": [
|
|
{
|
|
"format": "ms",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": 0,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"repeat": null,
|
|
"repeatIteration": null,
|
|
"repeatRowId": null,
|
|
"showTitle": true,
|
|
"title": "Memcached",
|
|
"titleSize": "h6"
|
|
},
|
|
{
|
|
"collapse": false,
|
|
"height": "250px",
|
|
"panels": [
|
|
{
|
|
"aliasColors": {
|
|
"1xx": "#EAB839",
|
|
"2xx": "#7EB26D",
|
|
"3xx": "#6ED0E0",
|
|
"4xx": "#EF843C",
|
|
"5xx": "#E24D42",
|
|
"OK": "#7EB26D",
|
|
"cancel": "#A9A9A9",
|
|
"error": "#E24D42",
|
|
"success": "#7EB26D"
|
|
},
|
|
"datasource": "$datasource",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 1,
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "ops"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"fill": 10,
|
|
"id": 16,
|
|
"linewidth": 0,
|
|
"links": [
|
|
|
|
],
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"span": 6,
|
|
"stack": true,
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z_]+)\"))\n",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"legendFormat": "{{status}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "QPS",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": "$datasource",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 1,
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "ms"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"id": 17,
|
|
"links": [
|
|
|
|
],
|
|
"nullPointMode": "null as zero",
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"span": 6,
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(0.99, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__rate_interval])) by (le,)) * 1e3",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} 99th",
|
|
"refId": "A",
|
|
"step": 10
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.50, sum(rate(tempodb_backend_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__rate_interval])) by (le,)) * 1e3",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} 50th",
|
|
"refId": "B",
|
|
"step": 10
|
|
},
|
|
{
|
|
"expr": "sum(rate(tempodb_backend_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempodb_backend_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\",operation=\"GET\"}[$__rate_interval])) by ()",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} Average",
|
|
"refId": "C",
|
|
"step": 10
|
|
}
|
|
],
|
|
"title": "Latency",
|
|
"type": "timeseries",
|
|
"yaxes": [
|
|
{
|
|
"format": "ms",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": 0,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"repeat": null,
|
|
"repeatIteration": null,
|
|
"repeatRowId": null,
|
|
"showTitle": true,
|
|
"title": "Backend",
|
|
"titleSize": "h6"
|
|
},
|
|
{
|
|
"collapse": false,
|
|
"height": "250px",
|
|
"panels": [
|
|
{
|
|
"aliasColors": {
|
|
"1xx": "#EAB839",
|
|
"2xx": "#7EB26D",
|
|
"3xx": "#6ED0E0",
|
|
"4xx": "#EF843C",
|
|
"5xx": "#E24D42",
|
|
"OK": "#7EB26D",
|
|
"cancel": "#A9A9A9",
|
|
"error": "#E24D42",
|
|
"success": "#7EB26D"
|
|
},
|
|
"datasource": "$datasource",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 1,
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"mode": "normal"
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "ops"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"fill": 10,
|
|
"id": 18,
|
|
"linewidth": 0,
|
|
"links": [
|
|
|
|
],
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"span": 6,
|
|
"stack": true,
|
|
"targets": [
|
|
{
|
|
"expr": "sum by (status) (\n label_replace(label_replace(rate(tempo_querier_external_endpoint_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z_]+)\"))\n",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"legendFormat": "{{status}}",
|
|
"refId": "A"
|
|
}
|
|
],
|
|
"title": "QPS",
|
|
"type": "timeseries"
|
|
},
|
|
{
|
|
"datasource": "$datasource",
|
|
"fieldConfig": {
|
|
"defaults": {
|
|
"custom": {
|
|
"drawStyle": "line",
|
|
"fillOpacity": 1,
|
|
"lineWidth": 1,
|
|
"pointSize": 5,
|
|
"showPoints": "never",
|
|
"spanNulls": false,
|
|
"stacking": {
|
|
"group": "A",
|
|
"mode": "none"
|
|
}
|
|
},
|
|
"thresholds": {
|
|
"mode": "absolute",
|
|
"steps": [
|
|
|
|
]
|
|
},
|
|
"unit": "ms"
|
|
},
|
|
"overrides": [
|
|
|
|
]
|
|
},
|
|
"id": 19,
|
|
"links": [
|
|
|
|
],
|
|
"nullPointMode": "null as zero",
|
|
"options": {
|
|
"legend": {
|
|
"showLegend": true
|
|
},
|
|
"tooltip": {
|
|
"mode": "single",
|
|
"sort": "none"
|
|
}
|
|
},
|
|
"span": 6,
|
|
"targets": [
|
|
{
|
|
"expr": "histogram_quantile(0.99, sum(rate(tempo_querier_external_endpoint_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__rate_interval])) by (le,)) * 1e3",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} 99th",
|
|
"refId": "A",
|
|
"step": 10
|
|
},
|
|
{
|
|
"expr": "histogram_quantile(0.50, sum(rate(tempo_querier_external_endpoint_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__rate_interval])) by (le,)) * 1e3",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} 50th",
|
|
"refId": "B",
|
|
"step": 10
|
|
},
|
|
{
|
|
"expr": "sum(rate(tempo_querier_external_endpoint_request_duration_seconds_sum{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__rate_interval])) by () * 1e3 / sum(rate(tempo_querier_external_endpoint_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/querier\"}[$__rate_interval])) by ()",
|
|
"format": "time_series",
|
|
"interval": "1m",
|
|
"intervalFactor": 2,
|
|
"legendFormat": "{{route}} Average",
|
|
"refId": "C",
|
|
"step": 10
|
|
}
|
|
],
|
|
"title": "Latency",
|
|
"type": "timeseries",
|
|
"yaxes": [
|
|
{
|
|
"format": "ms",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": 0,
|
|
"show": true
|
|
},
|
|
{
|
|
"format": "short",
|
|
"label": null,
|
|
"logBase": 1,
|
|
"max": null,
|
|
"min": null,
|
|
"show": false
|
|
}
|
|
]
|
|
}
|
|
],
|
|
"repeat": null,
|
|
"repeatIteration": null,
|
|
"repeatRowId": null,
|
|
"showTitle": true,
|
|
"title": "External Requests",
|
|
"titleSize": "h6"
|
|
}
|
|
],
|
|
"schemaVersion": 14,
|
|
"style": "dark",
|
|
"tags": [
|
|
"tempo"
|
|
],
|
|
"templating": {
|
|
"list": [
|
|
{
|
|
"current": {
|
|
"text": "default",
|
|
"value": "default"
|
|
},
|
|
"hide": 0,
|
|
"label": "Data source",
|
|
"name": "datasource",
|
|
"options": [
|
|
|
|
],
|
|
"query": "prometheus",
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"type": "datasource"
|
|
},
|
|
{
|
|
"allValue": null,
|
|
"current": {
|
|
"selected": true,
|
|
"text": "All",
|
|
"value": "$__all"
|
|
},
|
|
"datasource": "$datasource",
|
|
"hide": 0,
|
|
"includeAll": true,
|
|
"label": "cluster",
|
|
"multi": true,
|
|
"name": "cluster",
|
|
"options": [
|
|
|
|
],
|
|
"query": "label_values(tempo_build_info, cluster)",
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"sort": 2,
|
|
"tagValuesQuery": "",
|
|
"tags": [
|
|
|
|
],
|
|
"tagsQuery": "",
|
|
"type": "query",
|
|
"useTags": false
|
|
},
|
|
{
|
|
"allValue": null,
|
|
"current": {
|
|
"selected": true,
|
|
"text": "All",
|
|
"value": "$__all"
|
|
},
|
|
"datasource": "$datasource",
|
|
"hide": 0,
|
|
"includeAll": true,
|
|
"label": "namespace",
|
|
"multi": true,
|
|
"name": "namespace",
|
|
"options": [
|
|
|
|
],
|
|
"query": "label_values(tempo_build_info{cluster=~'$cluster'}, namespace)",
|
|
"refresh": 1,
|
|
"regex": "",
|
|
"sort": 2,
|
|
"tagValuesQuery": "",
|
|
"tags": [
|
|
|
|
],
|
|
"tagsQuery": "",
|
|
"type": "query",
|
|
"useTags": false
|
|
}
|
|
]
|
|
},
|
|
"time": {
|
|
"from": "now-1h",
|
|
"to": "now"
|
|
},
|
|
"timepicker": {
|
|
"refresh_intervals": [
|
|
"5s",
|
|
"10s",
|
|
"30s",
|
|
"1m",
|
|
"5m",
|
|
"15m",
|
|
"30m",
|
|
"1h",
|
|
"2h",
|
|
"1d"
|
|
],
|
|
"time_options": [
|
|
"5m",
|
|
"15m",
|
|
"1h",
|
|
"6h",
|
|
"12h",
|
|
"24h",
|
|
"2d",
|
|
"7d",
|
|
"30d"
|
|
]
|
|
},
|
|
"timezone": "utc",
|
|
"title": "Tempo / Reads",
|
|
"uid": "",
|
|
"version": 0
|
|
} |