mirror of
https://github.com/hanzoai/metrics.git
synced 2026-10-30 20:03:39 +00:00
Aliaksandr Valialkin
f1876a9b00
These metrics must be incremented when the request couldn't be processed because of the configured per-user concurrency limit.
The commit 14c10be8d4 moved the counter increase to the place when the current request
is put in the wait queue because of the concurrency limit is reached. This is incorrect, since such requests
can still be successfully processed during -maxQueueDuration . This also contradicts the docs at https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting
There is a small practical sense in counting the number of times the concurrency limit is reached,
while the request is successfully processed during the -maxQueueDuration after that.
Add missing alerting rule for rejected unauthorized requests because of the concurrency limit.
Add missing grouping by instance for per-user counter of rejected queries because of the concurrency limit.
Updates https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10078
59 lines
3.5 KiB
YAML
59 lines
3.5 KiB
YAML
# File contains default list of alerts for vmauth service.
|
|
# The alerts below are just recommendations and may require some updates
|
|
# and threshold calibration according to every specific setup.
|
|
groups:
|
|
- name: vmauth
|
|
interval: 30s
|
|
rules:
|
|
- alert: ConcurrentRequestsLimitReached
|
|
expr: sum(increase(vmauth_concurrent_requests_limit_reached_total[1m])) by (instance) > 0
|
|
for: 3m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
dashboard: "http://localhost:3000/d/nbuo5Mr4k?viewPanel=10&var-instance={{ $labels.instance }}"
|
|
summary: "vmauth ({{ $labels.instance }}) reached concurrent requests limit"
|
|
description: "Possible solutions: increase -maxQueueDuration flag value, increase -maxConcurrentRequests flag value,
|
|
deploy additional vmauth replicas, check requests latency at backend service.
|
|
See more details at https://docs.victoriametrics.com/victoriametrics/vmauth/#concurrency-limiting"
|
|
- alert: UserConcurrentRequestsLimitReached
|
|
expr: sum(increase(vmauth_user_concurrent_requests_limit_reached_total[1m])) by (instance, username) > 0
|
|
for: 3m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
dashboard: "http://localhost:3000/d/nbuo5Mr4k?viewPanel=10&var-instance={{ $labels.instance }}"
|
|
summary: "vmauth ({{ $labels.instance }}) has reached concurrent requests limit for username {{ $labels.username }}"
|
|
description: "Possible solutions: increase -maxQueueDuration flag value, increase -maxConcurrentPerUserRequests flag value,
|
|
deploy additional vmauth replicas, check requests latency at backend service."
|
|
- alert: UnauthorizedUserConcurrentRequestsLimitReached
|
|
expr: sum(increase(vmauth_unauthorized_user_concurrent_requests_limit_reached_total[1m])) by (instance) > 0
|
|
for: 3m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
dashboard: "http://localhost:3000/d/nbuo5Mr4k?viewPanel=10&var-instance={{ $labels.instance }}"
|
|
summary: "vmauth ({{ $labels.instance }}) has reached concurrent requests limit for unauthorized user"
|
|
description: "Possible solutions: increase -maxQueueDuration flag value, increase -maxConcurrentPerUserRequests flag value,
|
|
deploy additional vmauth replicas, check requests latency at backend service."
|
|
- alert: UnauthorizedUserRequestErrors
|
|
expr: increase(vmauth_unauthorized_user_request_errors_total[5m]) > 0
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
dashboard: "http://localhost:3000/d/nbuo5Mr4k?viewPanel=37&var-instance={{ $labels.instance }}"
|
|
summary: "Too many errors served for unauthorized user (instance {{ $labels.instance }})"
|
|
description: "Requests from unauthorized user are receiving errors.
|
|
Please check the vmauth logs to verify that the configuration is correct and clients are sending valid requests."
|
|
- alert: UserRequestErrors
|
|
expr: increase(vmauth_user_request_errors_total[5m]) > 0
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
dashboard: "http://localhost:3000/d/nbuo5Mr4k?viewPanel=37&var-instance={{ $labels.instance }}"
|
|
summary: "Too many errors served for user {{ $labels.username }} (instance {{ $labels.instance }})"
|
|
description: "Requests from user {{ $labels.username }} are receiving errors.
|
|
Please check the vmauth logs to verify that the configuration is correct and clients are sending valid requests."
|