mirror of
https://github.com/hanzos3/madmin-go.git
synced 2026-10-30 20:35:32 +00:00
863 lines
25 KiB
Go
863 lines
25 KiB
Go
//
|
|
// Copyright (c) 2015-2024 MinIO, Inc.
|
|
//
|
|
// This file is part of MinIO Object Storage stack
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Affero General Public License as
|
|
// published by the Free Software Foundation, either version 3 of the
|
|
// License, or (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Affero General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
//
|
|
|
|
package madmin
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"runtime/metrics"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/prometheus/procfs"
|
|
"github.com/shirou/gopsutil/v3/cpu"
|
|
"github.com/shirou/gopsutil/v3/load"
|
|
)
|
|
|
|
//msgp:clearomitted
|
|
//msgp:tag json
|
|
//go:generate msgp -unexported
|
|
|
|
// MetricType is a bitfield representation of different metric types.
|
|
type MetricType uint32
|
|
|
|
// MetricsNone indicates no metrics.
|
|
const MetricsNone MetricType = 0
|
|
|
|
const (
|
|
MetricsScanner MetricType = 1 << (iota)
|
|
MetricsDisk
|
|
MetricsOS
|
|
MetricsBatchJobs
|
|
MetricsSiteResync
|
|
MetricNet
|
|
MetricsMem
|
|
MetricsCPU
|
|
MetricsRPC
|
|
MetricsRuntime
|
|
|
|
// MetricsAll must be last.
|
|
// Enables all metrics.
|
|
MetricsAll = 1<<(iota) - 1
|
|
)
|
|
|
|
// MetricsOptions are options provided to Metrics call.
|
|
type MetricsOptions struct {
|
|
Type MetricType // Return only these metric types. Several types can be combined using |. Leave at 0 to return all.
|
|
N int // Maximum number of samples to return. 0 will return endless stream.
|
|
Interval time.Duration // Interval between samples. Will be rounded up to 1s.
|
|
Hosts []string // Leave empty for all
|
|
ByHost bool // Return metrics by host.
|
|
Disks []string
|
|
ByDisk bool
|
|
ByJobID string
|
|
ByDepID string
|
|
}
|
|
|
|
// Metrics makes an admin call to retrieve metrics.
|
|
// The provided function is called for each received entry.
|
|
func (adm *AdminClient) Metrics(ctx context.Context, o MetricsOptions, out func(RealtimeMetrics)) (err error) {
|
|
path := fmt.Sprintf(adminAPIPrefix + "/metrics")
|
|
q := make(url.Values)
|
|
q.Set("types", strconv.FormatUint(uint64(o.Type), 10))
|
|
q.Set("n", strconv.Itoa(o.N))
|
|
q.Set("interval", o.Interval.String())
|
|
q.Set("hosts", strings.Join(o.Hosts, ","))
|
|
if o.ByHost {
|
|
q.Set("by-host", "true")
|
|
}
|
|
q.Set("disks", strings.Join(o.Disks, ","))
|
|
if o.ByDisk {
|
|
q.Set("by-disk", "true")
|
|
}
|
|
if o.ByJobID != "" {
|
|
q.Set("by-jobID", o.ByJobID)
|
|
}
|
|
if o.ByDepID != "" {
|
|
q.Set("by-depID", o.ByDepID)
|
|
}
|
|
|
|
resp, err := adm.executeMethod(ctx,
|
|
http.MethodGet, requestData{
|
|
relPath: path,
|
|
queryValues: q,
|
|
},
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
closeResponse(resp)
|
|
return httpRespToErrorResponse(resp)
|
|
}
|
|
defer closeResponse(resp)
|
|
dec := json.NewDecoder(resp.Body)
|
|
for {
|
|
var m RealtimeMetrics
|
|
err := dec.Decode(&m)
|
|
if err != nil {
|
|
if errors.Is(err, io.EOF) {
|
|
err = io.ErrUnexpectedEOF
|
|
}
|
|
return err
|
|
}
|
|
out(m)
|
|
if m.Final {
|
|
break
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Contains returns whether m contains all of x.
|
|
func (m MetricType) Contains(x MetricType) bool {
|
|
return m&x == x
|
|
}
|
|
|
|
// RealtimeMetrics provides realtime metrics.
|
|
// This is intended to be expanded over time to cover more types.
|
|
type RealtimeMetrics struct {
|
|
// Error indicates an error occurred.
|
|
Errors []string `json:"errors,omitempty"`
|
|
// Hosts indicates the scanned hosts
|
|
Hosts []string `json:"hosts"`
|
|
Aggregated Metrics `json:"aggregated"`
|
|
ByHost map[string]Metrics `json:"by_host,omitempty"`
|
|
ByDisk map[string]DiskMetric `json:"by_disk,omitempty"`
|
|
// Final indicates whether this is the final packet and the receiver can exit.
|
|
Final bool `json:"final"`
|
|
}
|
|
|
|
// Metrics contains all metric types.
|
|
type Metrics struct {
|
|
Scanner *ScannerMetrics `json:"scanner,omitempty"`
|
|
Disk *DiskMetric `json:"disk,omitempty"`
|
|
OS *OSMetrics `json:"os,omitempty"`
|
|
BatchJobs *BatchJobMetrics `json:"batchJobs,omitempty"`
|
|
SiteResync *SiteResyncMetrics `json:"siteResync,omitempty"`
|
|
Net *NetMetrics `json:"net,omitempty"`
|
|
Mem *MemMetrics `json:"mem,omitempty"`
|
|
CPU *CPUMetrics `json:"cpu,omitempty"`
|
|
RPC *RPCMetrics `json:"rpc,omitempty"`
|
|
Go *RuntimeMetrics `json:"go,omitempty"`
|
|
}
|
|
|
|
// Merge other into r.
|
|
func (r *Metrics) Merge(other *Metrics) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
if r.Scanner == nil && other.Scanner != nil {
|
|
r.Scanner = &ScannerMetrics{}
|
|
}
|
|
r.Scanner.Merge(other.Scanner)
|
|
|
|
if r.Disk == nil && other.Disk != nil {
|
|
r.Disk = &DiskMetric{}
|
|
}
|
|
r.Disk.Merge(other.Disk)
|
|
|
|
if r.OS == nil && other.OS != nil {
|
|
r.OS = &OSMetrics{}
|
|
}
|
|
r.OS.Merge(other.OS)
|
|
if r.BatchJobs == nil && other.BatchJobs != nil {
|
|
r.BatchJobs = &BatchJobMetrics{}
|
|
}
|
|
r.BatchJobs.Merge(other.BatchJobs)
|
|
|
|
if r.SiteResync == nil && other.SiteResync != nil {
|
|
r.SiteResync = &SiteResyncMetrics{}
|
|
}
|
|
r.SiteResync.Merge(other.SiteResync)
|
|
|
|
if r.Net == nil && other.Net != nil {
|
|
r.Net = &NetMetrics{}
|
|
}
|
|
r.Net.Merge(other.Net)
|
|
if r.RPC == nil && other.RPC != nil {
|
|
r.RPC = &RPCMetrics{}
|
|
}
|
|
r.RPC.Merge(other.RPC)
|
|
if r.Go == nil && other.Go != nil {
|
|
r.Go = &RuntimeMetrics{}
|
|
}
|
|
r.Go.Merge(other.Go)
|
|
}
|
|
|
|
// Merge will merge other into r.
|
|
func (r *RealtimeMetrics) Merge(other *RealtimeMetrics) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
|
|
if len(other.Errors) > 0 {
|
|
r.Errors = append(r.Errors, other.Errors...)
|
|
}
|
|
|
|
if r.ByHost == nil && len(other.ByHost) > 0 {
|
|
r.ByHost = make(map[string]Metrics, len(other.ByHost))
|
|
}
|
|
for host, metrics := range other.ByHost {
|
|
r.ByHost[host] = metrics
|
|
}
|
|
|
|
r.Hosts = append(r.Hosts, other.Hosts...)
|
|
r.Aggregated.Merge(&other.Aggregated)
|
|
sort.Strings(r.Hosts)
|
|
|
|
// Gather per disk metrics
|
|
if r.ByDisk == nil && len(other.ByDisk) > 0 {
|
|
r.ByDisk = make(map[string]DiskMetric, len(other.ByDisk))
|
|
}
|
|
for disk, metrics := range other.ByDisk {
|
|
r.ByDisk[disk] = metrics
|
|
}
|
|
}
|
|
|
|
// ScannerMetrics contains scanner information.
|
|
type ScannerMetrics struct {
|
|
// Time these metrics were collected
|
|
CollectedAt time.Time `json:"collected"`
|
|
|
|
CurrentCycle uint64 `json:"current_cycle"` // Deprecated Mar 2024
|
|
CurrentStarted time.Time `json:"current_started"` // Deprecated Mar 2024
|
|
CyclesCompletedAt []time.Time `json:"cycle_complete_times"` // Deprecated Mar 2024
|
|
|
|
// Number of buckets currently scanning
|
|
OngoingBuckets int `json:"ongoing_buckets"`
|
|
|
|
// Stats per bucket, a map between bucket name and scan stats in all erasure sets
|
|
PerBucketStats map[string][]BucketScanInfo `json:"per_bucket_stats,omitempty"`
|
|
|
|
// Number of accumulated operations by type since server restart.
|
|
LifeTimeOps map[string]uint64 `json:"life_time_ops,omitempty"`
|
|
|
|
// Number of accumulated ILM operations by type since server restart.
|
|
LifeTimeILM map[string]uint64 `json:"ilm_ops,omitempty"`
|
|
|
|
// Last minute operation statistics.
|
|
LastMinute struct {
|
|
// Scanner actions.
|
|
Actions map[string]TimedAction `json:"actions,omitempty"`
|
|
// ILM actions.
|
|
ILM map[string]TimedAction `json:"ilm,omitempty"`
|
|
} `json:"last_minute"`
|
|
|
|
// Currently active path(s) being scanned.
|
|
ActivePaths []string `json:"active,omitempty"`
|
|
}
|
|
|
|
// Merge other into 's'.
|
|
func (s *ScannerMetrics) Merge(other *ScannerMetrics) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
|
|
if s.CollectedAt.Before(other.CollectedAt) {
|
|
// Use latest timestamp
|
|
s.CollectedAt = other.CollectedAt
|
|
}
|
|
|
|
if s.OngoingBuckets < other.OngoingBuckets {
|
|
s.OngoingBuckets = other.OngoingBuckets
|
|
}
|
|
|
|
if s.PerBucketStats == nil {
|
|
s.PerBucketStats = make(map[string][]BucketScanInfo)
|
|
}
|
|
for bucket, otherSt := range other.PerBucketStats {
|
|
if len(otherSt) == 0 {
|
|
continue
|
|
}
|
|
_, ok := s.PerBucketStats[bucket]
|
|
if !ok {
|
|
s.PerBucketStats[bucket] = otherSt
|
|
}
|
|
}
|
|
|
|
if s.CurrentCycle < other.CurrentCycle {
|
|
s.CurrentCycle = other.CurrentCycle
|
|
s.CyclesCompletedAt = other.CyclesCompletedAt
|
|
s.CurrentStarted = other.CurrentStarted
|
|
}
|
|
if len(other.CyclesCompletedAt) > len(s.CyclesCompletedAt) {
|
|
s.CyclesCompletedAt = other.CyclesCompletedAt
|
|
}
|
|
|
|
// Regular ops
|
|
if len(other.LifeTimeOps) > 0 && s.LifeTimeOps == nil {
|
|
s.LifeTimeOps = make(map[string]uint64, len(other.LifeTimeOps))
|
|
}
|
|
for k, v := range other.LifeTimeOps {
|
|
total := s.LifeTimeOps[k] + v
|
|
s.LifeTimeOps[k] = total
|
|
}
|
|
if s.LastMinute.Actions == nil && len(other.LastMinute.Actions) > 0 {
|
|
s.LastMinute.Actions = make(map[string]TimedAction, len(other.LastMinute.Actions))
|
|
}
|
|
for k, v := range other.LastMinute.Actions {
|
|
total := s.LastMinute.Actions[k]
|
|
total.Merge(v)
|
|
s.LastMinute.Actions[k] = total
|
|
}
|
|
|
|
// ILM
|
|
if len(other.LifeTimeILM) > 0 && s.LifeTimeILM == nil {
|
|
s.LifeTimeILM = make(map[string]uint64, len(other.LifeTimeILM))
|
|
}
|
|
for k, v := range other.LifeTimeILM {
|
|
total := s.LifeTimeILM[k] + v
|
|
s.LifeTimeILM[k] = total
|
|
}
|
|
if s.LastMinute.ILM == nil && len(other.LastMinute.ILM) > 0 {
|
|
s.LastMinute.ILM = make(map[string]TimedAction, len(other.LastMinute.ILM))
|
|
}
|
|
for k, v := range other.LastMinute.ILM {
|
|
total := s.LastMinute.ILM[k]
|
|
total.Merge(v)
|
|
s.LastMinute.ILM[k] = total
|
|
}
|
|
s.ActivePaths = append(s.ActivePaths, other.ActivePaths...)
|
|
sort.Strings(s.ActivePaths)
|
|
}
|
|
|
|
// DiskIOStats contains IO stats of a single drive
|
|
type DiskIOStats struct {
|
|
ReadIOs uint64 `json:"read_ios"`
|
|
ReadMerges uint64 `json:"read_merges"`
|
|
ReadSectors uint64 `json:"read_sectors"`
|
|
ReadTicks uint64 `json:"read_ticks"`
|
|
WriteIOs uint64 `json:"write_ios"`
|
|
WriteMerges uint64 `json:"write_merges"`
|
|
WriteSectors uint64 `json:"wrte_sectors"`
|
|
WriteTicks uint64 `json:"write_ticks"`
|
|
CurrentIOs uint64 `json:"current_ios"`
|
|
TotalTicks uint64 `json:"total_ticks"`
|
|
ReqTicks uint64 `json:"req_ticks"`
|
|
DiscardIOs uint64 `json:"discard_ios"`
|
|
DiscardMerges uint64 `json:"discard_merges"`
|
|
DiscardSectors uint64 `json:"discard_secotrs"`
|
|
DiscardTicks uint64 `json:"discard_ticks"`
|
|
FlushIOs uint64 `json:"flush_ios"`
|
|
FlushTicks uint64 `json:"flush_ticks"`
|
|
}
|
|
|
|
// DiskMetric contains metrics for one or more disks.
|
|
type DiskMetric struct {
|
|
// Time these metrics were collected
|
|
CollectedAt time.Time `json:"collected"`
|
|
|
|
// Number of disks
|
|
NDisks int `json:"n_disks"`
|
|
|
|
// Offline disks
|
|
Offline int `json:"offline,omitempty"`
|
|
|
|
// Healing disks
|
|
Healing int `json:"healing,omitempty"`
|
|
|
|
// Number of accumulated operations by type since server restart.
|
|
LifeTimeOps map[string]uint64 `json:"life_time_ops,omitempty"`
|
|
|
|
// Last minute statistics.
|
|
LastMinute struct {
|
|
Operations map[string]TimedAction `json:"operations,omitempty"`
|
|
} `json:"last_minute"`
|
|
|
|
IOStats DiskIOStats `json:"iostats,omitempty"`
|
|
}
|
|
|
|
// Merge other into 's'.
|
|
func (d *DiskMetric) Merge(other *DiskMetric) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
if d.CollectedAt.Before(other.CollectedAt) {
|
|
// Use latest timestamp
|
|
d.CollectedAt = other.CollectedAt
|
|
}
|
|
d.NDisks += other.NDisks
|
|
d.Offline += other.Offline
|
|
d.Healing += other.Healing
|
|
|
|
if len(other.LifeTimeOps) > 0 && d.LifeTimeOps == nil {
|
|
d.LifeTimeOps = make(map[string]uint64, len(other.LifeTimeOps))
|
|
}
|
|
for k, v := range other.LifeTimeOps {
|
|
total := d.LifeTimeOps[k] + v
|
|
d.LifeTimeOps[k] = total
|
|
}
|
|
|
|
if d.LastMinute.Operations == nil && len(other.LastMinute.Operations) > 0 {
|
|
d.LastMinute.Operations = make(map[string]TimedAction, len(other.LastMinute.Operations))
|
|
}
|
|
for k, v := range other.LastMinute.Operations {
|
|
total := d.LastMinute.Operations[k]
|
|
total.Merge(v)
|
|
d.LastMinute.Operations[k] = total
|
|
}
|
|
}
|
|
|
|
// OSMetrics contains metrics for OS operations.
|
|
type OSMetrics struct {
|
|
// Time these metrics were collected
|
|
CollectedAt time.Time `json:"collected"`
|
|
|
|
// Number of accumulated operations by type since server restart.
|
|
LifeTimeOps map[string]uint64 `json:"life_time_ops,omitempty"`
|
|
|
|
// Last minute statistics.
|
|
LastMinute struct {
|
|
Operations map[string]TimedAction `json:"operations,omitempty"`
|
|
} `json:"last_minute"`
|
|
}
|
|
|
|
// Merge other into 'o'.
|
|
func (o *OSMetrics) Merge(other *OSMetrics) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
if o.CollectedAt.Before(other.CollectedAt) {
|
|
// Use latest timestamp
|
|
o.CollectedAt = other.CollectedAt
|
|
}
|
|
|
|
if len(other.LifeTimeOps) > 0 && o.LifeTimeOps == nil {
|
|
o.LifeTimeOps = make(map[string]uint64, len(other.LifeTimeOps))
|
|
}
|
|
for k, v := range other.LifeTimeOps {
|
|
total := o.LifeTimeOps[k] + v
|
|
o.LifeTimeOps[k] = total
|
|
}
|
|
|
|
if o.LastMinute.Operations == nil && len(other.LastMinute.Operations) > 0 {
|
|
o.LastMinute.Operations = make(map[string]TimedAction, len(other.LastMinute.Operations))
|
|
}
|
|
for k, v := range other.LastMinute.Operations {
|
|
total := o.LastMinute.Operations[k]
|
|
total.Merge(v)
|
|
o.LastMinute.Operations[k] = total
|
|
}
|
|
}
|
|
|
|
// BatchJobMetrics contains metrics for batch operations
|
|
type BatchJobMetrics struct {
|
|
// Time these metrics were collected
|
|
CollectedAt time.Time `json:"collected"`
|
|
|
|
// Jobs by ID.
|
|
Jobs map[string]JobMetric
|
|
}
|
|
|
|
type JobMetric struct {
|
|
JobID string `json:"jobID"`
|
|
JobType string `json:"jobType"`
|
|
StartTime time.Time `json:"startTime"`
|
|
LastUpdate time.Time `json:"lastUpdate"`
|
|
RetryAttempts int `json:"retryAttempts"`
|
|
|
|
Complete bool `json:"complete"`
|
|
Failed bool `json:"failed"`
|
|
|
|
// Specific job type data:
|
|
Replicate *ReplicateInfo `json:"replicate,omitempty"`
|
|
KeyRotate *KeyRotationInfo `json:"rotation,omitempty"`
|
|
Expired *ExpirationInfo `json:"expired,omitempty"`
|
|
Catalog *CatalogInfo `json:"catalog,omitempty"`
|
|
}
|
|
|
|
type ReplicateInfo struct {
|
|
// Last bucket/object batch replicated
|
|
Bucket string `json:"lastBucket"`
|
|
Object string `json:"lastObject"`
|
|
|
|
// Verbose information
|
|
Objects int64 `json:"objects"`
|
|
ObjectsFailed int64 `json:"objectsFailed"`
|
|
DeleteMarkers int64 `json:"deleteMarkers"`
|
|
DeleteMarkersFailed int64 `json:"deleteMarkersFailed"`
|
|
BytesTransferred int64 `json:"bytesTransferred"`
|
|
BytesFailed int64 `json:"bytesFailed"`
|
|
}
|
|
|
|
type ExpirationInfo struct {
|
|
// Last bucket/object key rotated
|
|
Bucket string `json:"lastBucket"`
|
|
Object string `json:"lastObject"`
|
|
|
|
// Verbose information
|
|
Objects int64 `json:"objects"`
|
|
ObjectsFailed int64 `json:"objectsFailed"`
|
|
DeleteMarkers int64 `json:"deleteMarkers"`
|
|
DeleteMarkersFailed int64 `json:"deleteMarkersFailed"`
|
|
}
|
|
|
|
type KeyRotationInfo struct {
|
|
// Last bucket/object key rotated
|
|
Bucket string `json:"lastBucket"`
|
|
Object string `json:"lastObject"`
|
|
|
|
// Verbose information
|
|
Objects int64 `json:"objects"`
|
|
ObjectsFailed int64 `json:"objectsFailed"`
|
|
}
|
|
|
|
type CatalogInfo struct {
|
|
LastBucketScanned string `json:"lastBucketScanned"`
|
|
LastObjectScanned string `json:"lastObjectScanned"`
|
|
LastBucketMatched string `json:"lastBucketMatched"`
|
|
LastObjectMatched string `json:"lastObjectMatched"`
|
|
|
|
ObjectsScannedCount uint64 `json:"objectsScannedCount"`
|
|
ObjectsMatchedCount uint64 `json:"objectsMatchedCount"`
|
|
|
|
// Represents the number of objects' metadata that were written to output
|
|
// objects.
|
|
RecordsWrittenCount uint64 `json:"recordsWrittenCount"`
|
|
// Represents the number of output objects created.
|
|
OutputObjectsCount uint64 `json:"outputObjectsCount"`
|
|
// Manifest file path (part of the output of a catalog job)
|
|
ManifestPathBucket string `json:"manifestPathBucket"`
|
|
ManifestPathObject string `json:"manifestPathObject"`
|
|
|
|
// Error message
|
|
ErrorMsg string `json:"errorMsg"`
|
|
}
|
|
|
|
// Merge other into 'o'.
|
|
func (o *BatchJobMetrics) Merge(other *BatchJobMetrics) {
|
|
if other == nil || len(other.Jobs) == 0 {
|
|
return
|
|
}
|
|
if o.CollectedAt.Before(other.CollectedAt) {
|
|
// Use latest timestamp
|
|
o.CollectedAt = other.CollectedAt
|
|
}
|
|
if o.Jobs == nil {
|
|
o.Jobs = make(map[string]JobMetric, len(other.Jobs))
|
|
}
|
|
// Job
|
|
for k, v := range other.Jobs {
|
|
o.Jobs[k] = v
|
|
}
|
|
}
|
|
|
|
// SiteResyncMetrics contains metrics for site resync operation
|
|
type SiteResyncMetrics struct {
|
|
// Time these metrics were collected
|
|
CollectedAt time.Time `json:"collected"`
|
|
// Status of resync operation
|
|
ResyncStatus string `json:"resyncStatus,omitempty"`
|
|
StartTime time.Time `json:"startTime"`
|
|
LastUpdate time.Time `json:"lastUpdate"`
|
|
NumBuckets int64 `json:"numBuckets"`
|
|
ResyncID string `json:"resyncID"`
|
|
DeplID string `json:"deplID"`
|
|
|
|
// Completed size in bytes
|
|
ReplicatedSize int64 `json:"completedReplicationSize"`
|
|
// Total number of objects replicated
|
|
ReplicatedCount int64 `json:"replicationCount"`
|
|
// Failed size in bytes
|
|
FailedSize int64 `json:"failedReplicationSize"`
|
|
// Total number of failed operations
|
|
FailedCount int64 `json:"failedReplicationCount"`
|
|
// Buckets that could not be synced
|
|
FailedBuckets []string `json:"failedBuckets"`
|
|
// Last bucket/object replicated.
|
|
Bucket string `json:"bucket,omitempty"`
|
|
Object string `json:"object,omitempty"`
|
|
}
|
|
|
|
func (o SiteResyncMetrics) Complete() bool {
|
|
return strings.ToLower(o.ResyncStatus) == "completed"
|
|
}
|
|
|
|
// Merge other into 'o'.
|
|
func (o *SiteResyncMetrics) Merge(other *SiteResyncMetrics) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
if o.CollectedAt.Before(other.CollectedAt) {
|
|
// Use latest
|
|
*o = *other
|
|
}
|
|
}
|
|
|
|
type NetMetrics struct {
|
|
// Time these metrics were collected
|
|
CollectedAt time.Time `json:"collected"`
|
|
|
|
// net of Interface
|
|
InterfaceName string `json:"interfaceName"`
|
|
|
|
NetStats procfs.NetDevLine `json:"netstats"`
|
|
}
|
|
|
|
//msgp:replace procfs.NetDevLine with:procfsNetDevLine
|
|
|
|
// Merge other into 'o'.
|
|
func (n *NetMetrics) Merge(other *NetMetrics) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
if n.CollectedAt.Before(other.CollectedAt) {
|
|
// Use latest timestamp
|
|
n.CollectedAt = other.CollectedAt
|
|
}
|
|
n.NetStats.RxBytes += other.NetStats.RxBytes
|
|
n.NetStats.RxPackets += other.NetStats.RxPackets
|
|
n.NetStats.RxErrors += other.NetStats.RxErrors
|
|
n.NetStats.RxDropped += other.NetStats.RxDropped
|
|
n.NetStats.RxFIFO += other.NetStats.RxFIFO
|
|
n.NetStats.RxFrame += other.NetStats.RxFrame
|
|
n.NetStats.RxCompressed += other.NetStats.RxCompressed
|
|
n.NetStats.RxMulticast += other.NetStats.RxMulticast
|
|
n.NetStats.TxBytes += other.NetStats.TxBytes
|
|
n.NetStats.TxPackets += other.NetStats.TxPackets
|
|
n.NetStats.TxErrors += other.NetStats.TxErrors
|
|
n.NetStats.TxDropped += other.NetStats.TxDropped
|
|
n.NetStats.TxFIFO += other.NetStats.TxFIFO
|
|
n.NetStats.TxCollisions += other.NetStats.TxCollisions
|
|
n.NetStats.TxCarrier += other.NetStats.TxCarrier
|
|
n.NetStats.TxCompressed += other.NetStats.TxCompressed
|
|
}
|
|
|
|
//msgp:replace NodeCommon with:nodeCommon
|
|
|
|
// nodeCommon - use as replacement for NodeCommon
|
|
// We do not want to give NodeCommon codegen, since it is used for embedding.
|
|
type nodeCommon struct {
|
|
Addr string `json:"addr"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
// MemInfo contains system's RAM and swap information.
|
|
type MemInfo struct {
|
|
NodeCommon
|
|
|
|
Total uint64 `json:"total,omitempty"`
|
|
Used uint64 `json:"used,omitempty"`
|
|
Free uint64 `json:"free,omitempty"`
|
|
Available uint64 `json:"available,omitempty"`
|
|
Shared uint64 `json:"shared,omitempty"`
|
|
Cache uint64 `json:"cache,omitempty"`
|
|
Buffers uint64 `json:"buffer,omitempty"`
|
|
SwapSpaceTotal uint64 `json:"swap_space_total,omitempty"`
|
|
SwapSpaceFree uint64 `json:"swap_space_free,omitempty"`
|
|
// Limit will store cgroup limit if configured and
|
|
// less than Total, otherwise same as Total
|
|
Limit uint64 `json:"limit,omitempty"`
|
|
}
|
|
|
|
type MemMetrics struct {
|
|
// Time these metrics were collected
|
|
CollectedAt time.Time `json:"collected"`
|
|
|
|
Info MemInfo `json:"memInfo"`
|
|
}
|
|
|
|
// Merge other into 'm'.
|
|
func (m *MemMetrics) Merge(other *MemMetrics) {
|
|
if m.CollectedAt.Before(other.CollectedAt) {
|
|
// Use latest timestamp
|
|
m.CollectedAt = other.CollectedAt
|
|
}
|
|
|
|
m.Info.Total += other.Info.Total
|
|
m.Info.Available += other.Info.Available
|
|
m.Info.SwapSpaceTotal += other.Info.SwapSpaceTotal
|
|
m.Info.SwapSpaceFree += other.Info.SwapSpaceFree
|
|
m.Info.Limit += other.Info.Limit
|
|
}
|
|
|
|
//msgp:replace cpu.TimesStat with:cpuTimesStat
|
|
//msgp:replace load.AvgStat with:loadAvgStat
|
|
|
|
type CPUMetrics struct {
|
|
// Time these metrics were collected
|
|
CollectedAt time.Time `json:"collected"`
|
|
|
|
TimesStat *cpu.TimesStat `json:"timesStat"`
|
|
LoadStat *load.AvgStat `json:"loadStat"`
|
|
CPUCount int `json:"cpuCount"`
|
|
}
|
|
|
|
// Merge other into 'm'.
|
|
func (m *CPUMetrics) Merge(other *CPUMetrics) {
|
|
if m.CollectedAt.Before(other.CollectedAt) {
|
|
// Use latest timestamp
|
|
m.CollectedAt = other.CollectedAt
|
|
}
|
|
m.TimesStat.User += other.TimesStat.User
|
|
m.TimesStat.System += other.TimesStat.System
|
|
m.TimesStat.Idle += other.TimesStat.Idle
|
|
m.TimesStat.Nice += other.TimesStat.Nice
|
|
m.TimesStat.Iowait += other.TimesStat.Iowait
|
|
m.TimesStat.Irq += other.TimesStat.Irq
|
|
m.TimesStat.Softirq += other.TimesStat.Softirq
|
|
m.TimesStat.Steal += other.TimesStat.Steal
|
|
m.TimesStat.Guest += other.TimesStat.Guest
|
|
m.TimesStat.GuestNice += other.TimesStat.GuestNice
|
|
|
|
m.LoadStat.Load1 += other.LoadStat.Load1
|
|
m.LoadStat.Load5 += other.LoadStat.Load5
|
|
m.LoadStat.Load15 += other.LoadStat.Load15
|
|
}
|
|
|
|
// RPCMetrics contains metrics for RPC operations.
|
|
type RPCMetrics struct {
|
|
CollectedAt time.Time `json:"collectedAt"`
|
|
Connected int `json:"connected"`
|
|
ReconnectCount int `json:"reconnectCount"`
|
|
Disconnected int `json:"disconnected"`
|
|
OutgoingStreams int `json:"outgoingStreams"`
|
|
IncomingStreams int `json:"incomingStreams"`
|
|
OutgoingBytes int64 `json:"outgoingBytes"`
|
|
IncomingBytes int64 `json:"incomingBytes"`
|
|
OutgoingMessages int64 `json:"outgoingMessages"`
|
|
IncomingMessages int64 `json:"incomingMessages"`
|
|
OutQueue int `json:"outQueue"`
|
|
LastPongTime time.Time `json:"lastPongTime"`
|
|
LastPingMS float64 `json:"lastPingMS"`
|
|
MaxPingDurMS float64 `json:"maxPingDurMS"` // Maximum across all merged entries.
|
|
LastConnectTime time.Time `json:"lastConnectTime"`
|
|
|
|
ByDestination map[string]RPCMetrics `json:"byDestination,omitempty"`
|
|
ByCaller map[string]RPCMetrics `json:"byCaller,omitempty"`
|
|
}
|
|
|
|
// Merge other into 'm'.
|
|
func (m *RPCMetrics) Merge(other *RPCMetrics) {
|
|
if m == nil || other == nil {
|
|
return
|
|
}
|
|
if m.CollectedAt.Before(other.CollectedAt) {
|
|
// Use latest timestamp
|
|
m.CollectedAt = other.CollectedAt
|
|
}
|
|
if m.LastConnectTime.Before(other.LastConnectTime) {
|
|
m.LastConnectTime = other.LastConnectTime
|
|
}
|
|
m.Connected += other.Connected
|
|
m.Disconnected += other.Disconnected
|
|
m.ReconnectCount += other.ReconnectCount
|
|
m.OutgoingStreams += other.OutgoingStreams
|
|
m.IncomingStreams += other.IncomingStreams
|
|
m.OutgoingBytes += other.OutgoingBytes
|
|
m.IncomingBytes += other.IncomingBytes
|
|
m.OutgoingMessages += other.OutgoingMessages
|
|
m.IncomingMessages += other.IncomingMessages
|
|
m.OutQueue += other.OutQueue
|
|
if m.LastPongTime.Before(other.LastPongTime) {
|
|
m.LastPongTime = other.LastPongTime
|
|
m.LastPingMS = other.LastPingMS
|
|
}
|
|
if m.MaxPingDurMS < other.MaxPingDurMS {
|
|
m.MaxPingDurMS = other.MaxPingDurMS
|
|
}
|
|
for k, v := range other.ByDestination {
|
|
if m.ByDestination == nil {
|
|
m.ByDestination = make(map[string]RPCMetrics, len(other.ByDestination))
|
|
}
|
|
existing := m.ByDestination[k]
|
|
existing.Merge(&v)
|
|
m.ByDestination[k] = existing
|
|
}
|
|
for k, v := range other.ByCaller {
|
|
if m.ByCaller == nil {
|
|
m.ByCaller = make(map[string]RPCMetrics, len(other.ByCaller))
|
|
}
|
|
existing := m.ByCaller[k]
|
|
existing.Merge(&v)
|
|
m.ByCaller[k] = existing
|
|
}
|
|
}
|
|
|
|
//msgp:replace metrics.Float64Histogram with:localF64H
|
|
|
|
// local copy of localF64H, can be casted to/from metrics.Float64Histogram
|
|
type localF64H struct {
|
|
Counts []uint64 `json:"counts,omitempty"`
|
|
Buckets []float64 `json:"buckets,omitempty"`
|
|
}
|
|
|
|
// RuntimeMetrics contains metrics for the go runtime.
|
|
// See more at https://pkg.go.dev/runtime/metrics
|
|
type RuntimeMetrics struct {
|
|
// UintMetrics contains KindUint64 values
|
|
UintMetrics map[string]uint64 `json:"uintMetrics,omitempty"`
|
|
|
|
// FloatMetrics contains KindFloat64 values
|
|
FloatMetrics map[string]float64 `json:"floatMetrics,omitempty"`
|
|
|
|
// HistMetrics contains KindFloat64Histogram values
|
|
HistMetrics map[string]metrics.Float64Histogram `json:"histMetrics,omitempty"`
|
|
|
|
// N tracks the number of merged entries.
|
|
N int `json:"n"`
|
|
}
|
|
|
|
// Merge other into 'm'.
|
|
func (m *RuntimeMetrics) Merge(other *RuntimeMetrics) {
|
|
if m == nil || other == nil {
|
|
return
|
|
}
|
|
if m.UintMetrics == nil {
|
|
m.UintMetrics = make(map[string]uint64, len(other.UintMetrics))
|
|
}
|
|
if m.FloatMetrics == nil {
|
|
m.FloatMetrics = make(map[string]float64, len(other.FloatMetrics))
|
|
}
|
|
if m.HistMetrics == nil {
|
|
m.HistMetrics = make(map[string]metrics.Float64Histogram, len(other.HistMetrics))
|
|
}
|
|
for k, v := range other.UintMetrics {
|
|
m.UintMetrics[k] += v
|
|
}
|
|
for k, v := range other.FloatMetrics {
|
|
m.FloatMetrics[k] += v
|
|
}
|
|
for k, v := range other.HistMetrics {
|
|
existing := m.HistMetrics[k]
|
|
if len(existing.Buckets) == 0 {
|
|
m.HistMetrics[k] = v
|
|
continue
|
|
}
|
|
// TODO: Technically, I guess we may have differing buckets,
|
|
// but they should be the same for the runtime.
|
|
if len(existing.Buckets) == len(v.Buckets) {
|
|
for i, count := range v.Counts {
|
|
existing.Counts[i] += count
|
|
}
|
|
}
|
|
}
|
|
m.N += other.N
|
|
}
|