mirror of
https://github.com/hanzo-ml/kserve.git
synced 2026-10-30 20:53:49 +00:00
746f04fbbc
Signed-off-by: Sergey Bekkerman <[email protected]> Signed-off-by: SeanCondon <[email protected]> Co-authored-by: SeanCondon <[email protected]>
276 lines
15 KiB
JSON
276 lines
15 KiB
JSON
{
|
||
"annotations": { "list": [] },
|
||
"editable": true,
|
||
"fiscalYearStartMonth": 0,
|
||
"graphTooltip": 2,
|
||
"links": [
|
||
{
|
||
"asDropdown": false,
|
||
"icon": "external link",
|
||
"includeVars": false,
|
||
"keepTime": true,
|
||
"tags": [],
|
||
"targetBlank": true,
|
||
"title": "Kserve – Prefix Caching",
|
||
"tooltip": "",
|
||
"type": "link",
|
||
"url": "https://github.com/llm-d/llm-d/blob/main/docs/monitoring/example-promQL-queries.md#path-c-prefix-caching"
|
||
}
|
||
],
|
||
"liveNow": false,
|
||
"panels": [
|
||
{
|
||
"gridPos": { "h": 4, "w": 24, "x": 0, "y": 0 },
|
||
"id": 1,
|
||
"options": {
|
||
"content": "<div style=\"font-size: 14px; line-height: 1.6;\">\n<h2 style=\"margin-top: 0;\">Prefix Caching</h2>\n<p>This dashboard shows <strong>vLLM prefix cache</strong>, <strong>EPP prefix indexer</strong>, and <strong>llm-d KV cache (kvcache)</strong> metrics.</p>\n<p><strong>Documentation:</strong><br>\n<a href=\"https://github.com/llm-d/llm-d/blob/main/docs/monitoring/example-promQL-queries.md#path-c-prefix-caching\" target=\"_blank\">llm-d – Prefix Caching</a> — example PromQL queries and monitoring guide.</p>\n</div>",
|
||
"mode": "html"
|
||
},
|
||
"title": "Reference",
|
||
"type": "text"
|
||
},
|
||
{
|
||
"collapsed": false,
|
||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 },
|
||
"id": 2,
|
||
"panels": [],
|
||
"title": "vLLM prefix cache",
|
||
"type": "row"
|
||
},
|
||
{
|
||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||
"description": "Overall prefix cache hit rate across all vLLM instances. Green ≥80%, yellow 50–80%, red <50%.",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"color": { "mode": "palette-classic" },
|
||
"mappings": [],
|
||
"max": 1,
|
||
"min": 0,
|
||
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 0.8 }] },
|
||
"unit": "percentunit"
|
||
}
|
||
},
|
||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 5 },
|
||
"id": 3,
|
||
"options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": true },
|
||
"pluginVersion": "10.0.0",
|
||
"targets": [
|
||
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(vllm:prefix_cache_hits_total[5m])) / sum(rate(vllm:prefix_cache_queries_total[5m]))", "refId": "A" }
|
||
],
|
||
"title": "Prefix cache hit rate (vLLM)",
|
||
"type": "gauge"
|
||
},
|
||
{
|
||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||
"description": "Prefix cache hit rate per vLLM pod.",
|
||
"fieldConfig": {
|
||
"defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "percentunit", "custom": { "lineWidth": 2, "pointSize": 6, "lineInterpolation": "smooth" } }
|
||
},
|
||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 5 },
|
||
"id": 4,
|
||
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true } },
|
||
"pluginVersion": "10.0.0",
|
||
"targets": [
|
||
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by(pod) (rate(vllm:prefix_cache_hits_total[5m])) / sum by(pod) (rate(vllm:prefix_cache_queries_total[5m]))", "legendFormat": "{{pod}}", "refId": "A" }
|
||
],
|
||
"title": "Per-instance hit rate (vLLM)",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||
"description": "KV cache usage percentage by pod and model. Metric name must match --kv-cache-usage-percentage-metric in LLM inference service config (e.g. vllm:kv_cache_usage_perc).",
|
||
"fieldConfig": {
|
||
"defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "max": 100, "min": 0, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }, { "color": "yellow", "value": 70 }, { "color": "red", "value": 90 }] }, "unit": "percent", "custom": { "lineWidth": 2, "fillOpacity": 22, "pointSize": 6, "lineInterpolation": "smooth" } }
|
||
},
|
||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 13 },
|
||
"id": 5,
|
||
"options": { "legend": { "displayMode": "table", "placement": "bottom", "showLegend": true } },
|
||
"pluginVersion": "10.0.0",
|
||
"targets": [
|
||
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "avg by(pod, model_name) (vllm:kv_cache_usage_perc * 100)", "legendFormat": "{{pod}} ({{model_name}})", "refId": "A" }
|
||
],
|
||
"title": "Cache utilization % (vLLM)",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"collapsed": false,
|
||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 },
|
||
"id": 7,
|
||
"panels": [],
|
||
"title": "EPP prefix indexer",
|
||
"type": "row"
|
||
},
|
||
{
|
||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||
"description": "EPP prefix indexer size.",
|
||
"fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "short", "custom": { "lineWidth": 2, "lineInterpolation": "smooth" } } },
|
||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 22 },
|
||
"id": 8,
|
||
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true } },
|
||
"pluginVersion": "10.0.0",
|
||
"targets": [
|
||
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "inference_extension_prefix_indexer_size", "legendFormat": "{{pod}}", "refId": "A" }
|
||
],
|
||
"title": "EPP prefix indexer size",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||
"description": "EPP prefix indexer hit ratio percentiles. Shows 0 when no EPP prefix hits in the selected window.",
|
||
"fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "percentunit", "custom": { "lineWidth": 2, "lineInterpolation": "smooth" } } },
|
||
"gridPos": { "h": 7, "w": 12, "x": 12, "y": 22 },
|
||
"id": 9,
|
||
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true } },
|
||
"pluginVersion": "10.0.0",
|
||
"targets": [
|
||
{ "expr": "histogram_quantile(0.50, sum by(le) (rate(inference_extension_prefix_indexer_hit_ratio_bucket[5m])))", "legendFormat": "P50", "refId": "A" },
|
||
{ "expr": "histogram_quantile(0.90, sum by(le) (rate(inference_extension_prefix_indexer_hit_ratio_bucket[5m])))", "legendFormat": "P90", "refId": "B" }
|
||
],
|
||
"title": "EPP prefix indexer hit ratio (P50, P90)",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||
"description": "EPP prefix indexer hit bytes percentiles. Shows 0 when no EPP prefix hits in the selected window.",
|
||
"fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "bytes", "custom": { "lineWidth": 2, "lineInterpolation": "smooth" } } },
|
||
"gridPos": { "h": 7, "w": 12, "x": 0, "y": 29 },
|
||
"id": 10,
|
||
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true } },
|
||
"pluginVersion": "10.0.0",
|
||
"targets": [
|
||
{ "expr": "histogram_quantile(0.50, sum by(le) (rate(inference_extension_prefix_indexer_hit_bytes_bucket[5m])))", "legendFormat": "P50", "refId": "A" },
|
||
{ "expr": "histogram_quantile(0.90, sum by(le) (rate(inference_extension_prefix_indexer_hit_bytes_bucket[5m])))", "legendFormat": "P90", "refId": "B" }
|
||
],
|
||
"title": "EPP prefix indexer hit bytes (P50, P90)",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"collapsed": false,
|
||
"gridPos": { "h": 1, "w": 24, "x": 0, "y": 36 },
|
||
"id": 11,
|
||
"panels": [],
|
||
"title": "llm-d KV cache (kvcache)",
|
||
"type": "row"
|
||
},
|
||
{
|
||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||
"description": "Overall KV-block lookup hit rate (keys found in cache / lookup calls). From kvcache_index_lookup_hits_total and kvcache_index_lookup_requests_total.",
|
||
"fieldConfig": {
|
||
"defaults": {
|
||
"color": { "mode": "palette-classic" },
|
||
"mappings": [],
|
||
"max": 1,
|
||
"min": 0,
|
||
"thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "yellow", "value": 0.5 }, { "color": "green", "value": 0.8 }] },
|
||
"unit": "percentunit"
|
||
}
|
||
},
|
||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 37 },
|
||
"id": 12,
|
||
"options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "showThresholdLabels": true },
|
||
"pluginVersion": "10.0.0",
|
||
"targets": [
|
||
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(kvcache_index_lookup_hits_total[5m])) / sum(rate(kvcache_index_lookup_requests_total[5m]))", "refId": "A" }
|
||
],
|
||
"title": "KV cache lookup hit rate",
|
||
"type": "gauge"
|
||
},
|
||
{
|
||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||
"description": "Rate of KV-block admissions and evictions.",
|
||
"fieldConfig": {
|
||
"defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "short", "custom": { "lineWidth": 2, "fillOpacity": 20, "lineInterpolation": "smooth" } },
|
||
"overrides": [
|
||
{ "matcher": { "id": "byName", "options": "admissions/s" }, "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] },
|
||
{ "matcher": { "id": "byName", "options": "evictions/s" }, "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] }
|
||
]
|
||
},
|
||
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 37 },
|
||
"id": 13,
|
||
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true } },
|
||
"pluginVersion": "10.0.0",
|
||
"targets": [
|
||
{ "expr": "sum(rate(kvcache_index_admissions_total[5m]))", "legendFormat": "admissions/s", "refId": "A" },
|
||
{ "expr": "sum(rate(kvcache_index_evictions_total[5m]))", "legendFormat": "evictions/s", "refId": "B" }
|
||
],
|
||
"title": "Admissions vs evictions (rate)",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||
"description": "Lookup call latency percentiles (kvcache_index_lookup_latency_seconds).",
|
||
"fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "s", "custom": { "lineWidth": 2, "lineInterpolation": "smooth" } } },
|
||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 43 },
|
||
"id": 15,
|
||
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true } },
|
||
"pluginVersion": "10.0.0",
|
||
"targets": [
|
||
{ "expr": "histogram_quantile(0.50, sum by(le) (rate(kvcache_index_lookup_latency_seconds_bucket[5m])))", "legendFormat": "P50", "refId": "A" },
|
||
{ "expr": "histogram_quantile(0.90, sum by(le) (rate(kvcache_index_lookup_latency_seconds_bucket[5m])))", "legendFormat": "P90", "refId": "B" },
|
||
{ "expr": "histogram_quantile(0.99, sum by(le) (rate(kvcache_index_lookup_latency_seconds_bucket[5m])))", "legendFormat": "P99", "refId": "C" }
|
||
],
|
||
"title": "Lookup latency (P50, P90, P99)",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||
"description": "Maximum cache hits on a single pod on Lookup() (rate).",
|
||
"fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "short", "custom": { "lineWidth": 2, "lineInterpolation": "smooth" } } },
|
||
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 43 },
|
||
"id": 16,
|
||
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true } },
|
||
"pluginVersion": "10.0.0",
|
||
"targets": [
|
||
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum(rate(kvcache_index_max_pod_hit_count_total[5m]))", "legendFormat": "max pod hit count/s", "refId": "A" }
|
||
],
|
||
"title": "Max pod hit count (rate)",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||
"description": "Tokenized tokens rate by tokenizer (kvcache_tokenization_tokenized_tokens_total).",
|
||
"fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "short", "custom": { "lineWidth": 2, "fillOpacity": 20, "lineInterpolation": "smooth" } } },
|
||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 49 },
|
||
"id": 17,
|
||
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true } },
|
||
"pluginVersion": "10.0.0",
|
||
"targets": [
|
||
{ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "expr": "sum by(tokenizer) (rate(kvcache_tokenization_tokenized_tokens_total[5m]))", "legendFormat": "{{tokenizer}}", "refId": "A" }
|
||
],
|
||
"title": "Tokenized tokens (rate by tokenizer)",
|
||
"type": "timeseries"
|
||
},
|
||
{
|
||
"datasource": { "type": "prometheus", "uid": "${datasource}" },
|
||
"description": "Render chat template and tokenization latency percentiles by tokenizer.",
|
||
"fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, "unit": "s", "custom": { "lineWidth": 2, "lineInterpolation": "smooth" } } },
|
||
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 49 },
|
||
"id": 18,
|
||
"options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true } },
|
||
"pluginVersion": "10.0.0",
|
||
"targets": [
|
||
{ "expr": "histogram_quantile(0.90, sum by(tokenizer, le) (rate(kvcache_tokenization_render_chat_template_latency_seconds_bucket[5m])))", "legendFormat": "render {{tokenizer}} P90", "refId": "A" },
|
||
{ "expr": "histogram_quantile(0.90, sum by(tokenizer, le) (rate(kvcache_tokenization_tokenization_latency_seconds_bucket[5m])))", "legendFormat": "tokenize {{tokenizer}} P90", "refId": "B" }
|
||
],
|
||
"title": "Tokenization latency P90 (by tokenizer)",
|
||
"type": "timeseries"
|
||
}
|
||
],
|
||
"refresh": "30s",
|
||
"schemaVersion": 38,
|
||
"style": "dark",
|
||
"tags": ["llm-d", "prefix-caching", "vllm", "epp", "kserve", "kvcache"],
|
||
"templating": {
|
||
"list": [
|
||
{ "current": { "selected": false, "text": "Prometheus", "value": "Prometheus" }, "hide": 0, "includeAll": false, "label": "Datasource", "multi": false, "name": "datasource", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }
|
||
]
|
||
},
|
||
"time": { "from": "now-1h", "to": "now" },
|
||
"timepicker": {},
|
||
"timezone": "",
|
||
"title": "Kserve EPP metrics – Prefix Caching",
|
||
"uid": "kserve-epp-prefix-caching",
|
||
"version": 1,
|
||
"weekStart": ""
|
||
}
|