mirror of
https://github.com/hanzoai/status.git
synced 2026-10-30 18:21:48 +00:00
zeekayandHanzo Dev
78ab4f2f83
zip's registration is a verb method on the group, so a route that is not a typed operation says which method it answers with. The security package's own two-method router goes with it: it existed so both the app and a sub-router could satisfy one name, and restating a routing surface is what let the two disagree — the methods it named no longer exist. The app spells itself as a group, which registers at exactly the paths it did. The toolchain and the pin move together, because below zip v1.36.50 the JSON encoder was selected on a build tag: a repo whose go directive rises first writes a wire that is neither the old release's nor the new one's until the pin catches up. metrics_test.go still does not compile, before and after: it compares a metric registry against a gatherer from a different metrics library, so the two Gather signatures disagree. That is its own change. Co-authored-by: Hanzo Dev <[email protected]>
84 lines
4.1 KiB
Go
84 lines
4.1 KiB
Go
package watchdog
|
|
|
|
import (
|
|
"context"
|
|
"time"
|
|
|
|
"github.com/TwiN/logr"
|
|
"hanzo.ai/status/config"
|
|
"hanzo.ai/status/config/endpoint"
|
|
"hanzo.ai/status/metrics"
|
|
"hanzo.ai/status/storage/store"
|
|
)
|
|
|
|
func monitorExternalEndpointHeartbeat(ee *endpoint.ExternalEndpoint, cfg *config.Config, extraLabels []string, ctx context.Context) {
|
|
ticker := time.NewTicker(ee.Heartbeat.Interval)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
logr.Warnf("[watchdog.monitorExternalEndpointHeartbeat] Canceling current execution of group=%s; endpoint=%s; key=%s", ee.Group, ee.Name, ee.Key())
|
|
return
|
|
case <-ticker.C:
|
|
executeExternalEndpointHeartbeat(ee, cfg, extraLabels)
|
|
}
|
|
}
|
|
}
|
|
|
|
func executeExternalEndpointHeartbeat(ee *endpoint.ExternalEndpoint, cfg *config.Config, extraLabels []string) {
|
|
// Acquire semaphore to limit concurrent external endpoint monitoring
|
|
if err := monitoringSemaphore.Acquire(ctx, 1); err != nil {
|
|
// Only fails if context is cancelled (during shutdown)
|
|
logr.Debugf("[watchdog.executeExternalEndpointHeartbeat] Context cancelled, skipping execution: %s", err.Error())
|
|
return
|
|
}
|
|
defer monitoringSemaphore.Release(1)
|
|
// If there's a connectivity checker configured, check if Gatus has internet connectivity
|
|
if cfg.Connectivity != nil && cfg.Connectivity.Checker != nil && !cfg.Connectivity.Checker.IsConnected() {
|
|
logr.Infof("[watchdog.monitorExternalEndpointHeartbeat] No connectivity; skipping execution")
|
|
return
|
|
}
|
|
logr.Debugf("[watchdog.monitorExternalEndpointHeartbeat] Checking heartbeat for group=%s; endpoint=%s; key=%s", ee.Group, ee.Name, ee.Key())
|
|
convertedEndpoint := ee.ToEndpoint()
|
|
hasReceivedResultWithinHeartbeatInterval, err := store.Get().HasEndpointStatusNewerThan(ee.Key(), time.Now().Add(-ee.Heartbeat.Interval))
|
|
if err != nil {
|
|
logr.Errorf("[watchdog.monitorExternalEndpointHeartbeat] Failed to check if endpoint has received a result within the heartbeat interval: %s", err.Error())
|
|
return
|
|
}
|
|
if hasReceivedResultWithinHeartbeatInterval {
|
|
// If we received a result within the heartbeat interval, we don't want to create a successful result, so we
|
|
// skip the rest. We don't have to worry about alerting or metrics, because if the previous heartbeat failed
|
|
// while this one succeeds, it implies that there was a new result pushed, and that result being pushed
|
|
// should've resolved the alert.
|
|
logr.Infof("[watchdog.monitorExternalEndpointHeartbeat] Checked heartbeat for group=%s; endpoint=%s; key=%s; success=%v; errors=%d", ee.Group, ee.Name, ee.Key(), hasReceivedResultWithinHeartbeatInterval, 0)
|
|
return
|
|
}
|
|
// All code after this point assumes the heartbeat failed
|
|
result := &endpoint.Result{
|
|
Timestamp: time.Now(),
|
|
Success: false,
|
|
Errors: []string{"heartbeat: no update received within " + ee.Heartbeat.Interval.String()},
|
|
}
|
|
if cfg.Metrics {
|
|
metrics.PublishMetricsForEndpoint(convertedEndpoint, result, extraLabels)
|
|
}
|
|
UpdateEndpointStatus(convertedEndpoint, result)
|
|
logr.Infof("[watchdog.monitorExternalEndpointHeartbeat] Checked heartbeat for group=%s; endpoint=%s; key=%s; success=%v; errors=%d; duration=%s", ee.Group, ee.Name, ee.Key(), result.Success, len(result.Errors), result.Duration.Round(time.Millisecond))
|
|
inEndpointMaintenanceWindow := false
|
|
for _, maintenanceWindow := range ee.MaintenanceWindows {
|
|
if maintenanceWindow.IsUnderMaintenance() {
|
|
logr.Debug("[watchdog.monitorExternalEndpointHeartbeat] Under endpoint maintenance window")
|
|
inEndpointMaintenanceWindow = true
|
|
}
|
|
}
|
|
if !cfg.Maintenance.IsUnderMaintenance() && !inEndpointMaintenanceWindow {
|
|
HandleAlerting(convertedEndpoint, result, cfg.Alerting)
|
|
// Sync the failure/success counters back to the external endpoint
|
|
ee.NumberOfSuccessesInARow = convertedEndpoint.NumberOfSuccessesInARow
|
|
ee.NumberOfFailuresInARow = convertedEndpoint.NumberOfFailuresInARow
|
|
} else {
|
|
logr.Debug("[watchdog.monitorExternalEndpointHeartbeat] Not handling alerting because currently in the maintenance window")
|
|
}
|
|
logr.Debugf("[watchdog.monitorExternalEndpointHeartbeat] Waiting for interval=%s before checking heartbeat for group=%s endpoint=%s (key=%s) again", ee.Heartbeat.Interval, ee.Group, ee.Name, ee.Key())
|
|
}
|