mirror of
https://github.com/hanzoai/metrics.git
synced 2026-10-30 20:03:39 +00:00
Note: remote write request won't fail immediately if `-remoteWrite.url` is unreachable, as vmalert maintains a remote write queue (with capacity controlled by `-remoteWrite.maxQueueSize(default 1e5)`) and uses a separate process to batch and push queued data. vmalert uses error group to print error messages associated with a single group together, which should assist the group owner in reviewing relevant error messages. With this pull request, the error message would be like: ``` 2026-01-30T08:26:46.641Z error app/vmalert/rule/group.go:395 group "group2": errors(3): rule "rule1": remote write failure: failed to push timeseries - queue is full (1 entries). Queue size is controlled by -remoteWrite.maxQueueSize flag rule "rule1": notifier failure: failed to send alerts to addr "http://non-existing-alertmanager-1/api/v2/alerts": invalid SC 502 from "http://non-existing-alertmanager-1/api/v2/alerts"; response body: rule "rule1": notifier failure: failed to send alerts to addr "http://non-existing-alertmanager-2/api/v2/alerts": invalid SC 502 from "http://non-existing-alertmanager-2/api/v2/alerts"; response body: 2026-01-30T08:26:46.641Z error app/vmalert/rule/group.go:395 group "group2": errors(3): rule "rule2": remote write failure: failed to push timeseries - queue is full (1 entries). Queue size is controlled by -remoteWrite.maxQueueSize flag rule "rule2": notifier failure: failed to send alerts to addr "http://non-existing-alertmanager-2/api/v2/alerts": invalid SC 502 from "http://non-existing-alertmanager-2/api/v2/alerts"; response body: rule "rule2": notifier failure: failed to send alerts to addr "http://non-existing-alertmanager-1/api/v2/alerts": invalid SC 502 from "http://non-existing-alertmanager-1/api/v2/alerts"; response body: 2026-01-30T08:26:52.229Z error app/vmalert/rule/group.go:395 group "group1": errors(3): rule "rule1": remote write failure: failed to push timeseries - queue is full (1 entries). Queue size is controlled by -remoteWrite.maxQueueSize flag rule "rule1": notifier failure: failed to send alerts to addr "http://non-existing-alertmanager-1/api/v2/alerts": invalid SC 502 from "http://non-existing-alertmanager-1/api/v2/alerts"; response body: rule "rule1": notifier failure: failed to send alerts to addr "http://non-existing-alertmanager-2/api/v2/alerts": invalid SC 502 from "http://non-existing-alertmanager-2/api/v2/alerts"; response body: 2026-01-30T08:26:52.229Z error app/vmalert/rule/group.go:395 group "group1": errors(3): rule "rule2": remote write failure: failed to push timeseries - queue is full (1 entries). Queue size is controlled by -remoteWrite.maxQueueSize flag rule "rule2": notifier failure: failed to send alerts to addr "http://non-existing-alertmanager-2/api/v2/alerts": invalid SC 502 from "http://non-existing-alertmanager-2/api/v2/alerts"; response body: rule "rule2": notifier failure: failed to send alerts to addr "http://non-existing-alertmanager-1/api/v2/alerts": invalid SC 502 from "http://non-existing-alertmanager-1/api/v2/alerts"; response body: ``` Related PR https://github.com/VictoriaMetrics/VictoriaMetrics/pull/10376
57 lines
919 B
Go
57 lines
919 B
Go
package vmalertutil
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
"sync"
|
|
)
|
|
|
|
// ErrGroup accumulates multiple errors
|
|
// and produces single error message.
|
|
type ErrGroup struct {
|
|
mu sync.Mutex
|
|
errs []error
|
|
}
|
|
|
|
// Add adds a new error to group.
|
|
// Is thread-safe.
|
|
func (eg *ErrGroup) Add(err error) {
|
|
eg.mu.Lock()
|
|
eg.errs = append(eg.errs, err)
|
|
eg.mu.Unlock()
|
|
}
|
|
|
|
// Err checks if group contains at least
|
|
// one error.
|
|
func (eg *ErrGroup) Err() error {
|
|
if eg == nil {
|
|
return nil
|
|
}
|
|
|
|
eg.mu.Lock()
|
|
defer eg.mu.Unlock()
|
|
if len(eg.errs) == 0 {
|
|
return nil
|
|
}
|
|
return eg
|
|
}
|
|
|
|
// Error satisfies Error interface
|
|
func (eg *ErrGroup) Error() string {
|
|
eg.mu.Lock()
|
|
defer eg.mu.Unlock()
|
|
|
|
if len(eg.errs) == 0 {
|
|
return ""
|
|
}
|
|
var b strings.Builder
|
|
fmt.Fprintf(&b, "errors(%d): \n", len(eg.errs))
|
|
for i, err := range eg.errs {
|
|
b.WriteString(err.Error())
|
|
if i != len(eg.errs)-1 {
|
|
b.WriteString("\n")
|
|
}
|
|
}
|
|
return b.String()
|
|
}
|