Files
zeekay 7c5a2985bc fix(metrics): serve DefaultRegistry at /v1/metrics (was empty-body 500)
GET api.hanzo.ai/v1/metrics 500'd with an empty body and Prometheus
content-type. The gateway proxies /v1/metrics to this service (Casdoor
fork); the 500 originated in ApiController.GetMetrics, not the gateway's
OpenCensus exporter.

Two defects made the endpoint wrong:

1. GetMetrics served metric.Handler(), which luxfi/metric implements as
   HandlerFor(NewRegistry()) — a fresh, throwaway registry. It NEVER sees
   the DefaultRegistry that object/prometheus.go records into, so the
   scrape is always empty. GetPrometheusInfo already reads DefaultRegistry;
   the two disagreed. Bind both to the one registry via a new
   object.MetricsHandler() = metric.HandlerFor(metric.DefaultRegistry).

2. luxfi/metric is no-op unless built with `-tags metrics` (its documented
   design). The Dockerfile built ./cmd/aid without the tag, so every metric
   was a no-op and DefaultRegistry gathered nothing. Add -tags metrics.

Verified end-to-end: curl 127.0.0.1/v1/metrics -> 200,
Content-Type text/plain; version=0.0.4; charset=utf-8, valid exposition
(cloud_cpu_usage / cloud_memory_usage / cloud_api_throughput). Regression
tests: TestMetricsHandlerContract (never-500 in any build) and, under
-tags metrics, TestMetricsHandlerServesDefaultRegistry (non-empty output).
2026-07-04 19:22:38 -07:00

143 lines
4.5 KiB
Go

// Copyright 2023-2025 Hanzo AI Inc. All Rights Reserved.
// Portions Copyright 2025 The OpenAgent Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package object
import (
"fmt"
"net/http"
"time"
metric "github.com/luxfi/metric"
)
type PrometheusInfo struct {
ApiThroughput []GaugeVecInfo `json:"apiThroughput"`
ApiLatency []HistogramVecInfo `json:"apiLatency"`
TotalThroughput float64 `json:"totalThroughput"`
}
type GaugeVecInfo struct {
Method string `json:"method"`
Name string `json:"name"`
Throughput float64 `json:"throughput"`
}
type HistogramVecInfo struct {
Method string `json:"method"`
Name string `json:"name"`
Count uint64 `json:"count"`
Latency string `json:"latency"`
}
var (
// ApiThroughput uses *metric.GaugeVec directly because Reset() is needed
ApiThroughput = metric.NewGaugeVec(metric.GaugeOpts{
Name: "cloud_api_throughput",
Help: "The throughput of each api access",
}, []string{"path", "method"})
ApiLatency = metric.NewHistogramVec(metric.HistogramOpts{
Name: "cloud_api_latency",
Help: "API processing latency in milliseconds",
}, []string{"path", "method"})
CpuUsage = metric.NewGaugeVec(metric.GaugeOpts{
Name: "cloud_cpu_usage",
Help: "Hanzo Cloud cpu usage",
}, []string{"cpuNum"})
MemoryUsage = metric.NewGaugeVec(metric.GaugeOpts{
Name: "cloud_memory_usage",
Help: "Hanzo Cloud memory usage in Byte",
}, []string{"type"})
TotalThroughput = metric.NewGauge(metric.GaugeOpts{
Name: "cloud_total_throughput",
Help: "The total throughput of Hanzo Cloud",
})
)
func ClearThroughputPerSecond() {
ticker := time.NewTicker(time.Second)
for range ticker.C {
ApiThroughput.Reset()
TotalThroughput.Set(0)
}
}
func GetPrometheusInfo() (*PrometheusInfo, error) {
res := &PrometheusInfo{}
metricFamilies, err := metric.DefaultRegistry.Gather()
if err != nil {
return nil, err
}
for _, metricFamily := range metricFamilies {
switch metricFamily.Name {
case "cloud_api_throughput":
res.ApiThroughput = getGaugeVecInfo(metricFamily)
case "cloud_api_latency":
res.ApiLatency = getHistogramVecInfo(metricFamily)
case "cloud_total_throughput":
if len(metricFamily.Metrics) > 0 {
res.TotalThroughput = metricFamily.Metrics[0].Value.Value
}
}
}
return res, nil
}
// MetricsHandler returns the Prometheus text-exposition handler for GET
// /v1/metrics, bound to the SAME DefaultRegistry the metric vars above register
// into (the package-level metric.NewGaugeVec / NewHistogramVec helpers delegate
// to DefaultRegistry). This must NOT use metric.Handler(): that convenience
// wrapper binds a fresh, throwaway registry, so it exposes an EMPTY scrape
// regardless of what has been recorded — the endpoint would answer 200 with a
// zero-length body forever. Binding DefaultRegistry mirrors GetPrometheusInfo,
// which reads the same registry: one registry, one source of truth.
func MetricsHandler() http.Handler {
return metric.HandlerFor(metric.DefaultRegistry)
}
func getHistogramVecInfo(metricFamily *metric.MetricFamily) []HistogramVecInfo {
var histogramVecInfos []HistogramVecInfo
for _, m := range metricFamily.Metrics {
sampleCount := m.Value.SampleCount
sampleSum := m.Value.SampleSum
latency := sampleSum / float64(sampleCount)
histogramVecInfo := HistogramVecInfo{
Method: labelValue(m.Labels, 0),
Name: labelValue(m.Labels, 1),
Count: sampleCount,
Latency: fmt.Sprintf("%.3f", latency),
}
histogramVecInfos = append(histogramVecInfos, histogramVecInfo)
}
return histogramVecInfos
}
func getGaugeVecInfo(metricFamily *metric.MetricFamily) []GaugeVecInfo {
var counterVecInfos []GaugeVecInfo
for _, m := range metricFamily.Metrics {
counterVecInfo := GaugeVecInfo{
Method: labelValue(m.Labels, 0),
Name: labelValue(m.Labels, 1),
Throughput: m.Value.Value,
}
counterVecInfos = append(counterVecInfos, counterVecInfo)
}
return counterVecInfos
}
func labelValue(labels []metric.LabelPair, i int) string {
if i < 0 || i >= len(labels) {
return ""
}
return labels[i].Value
}