mirror of
https://github.com/hanzos3/madmin-go.git
synced 2026-10-30 20:35:32 +00:00
* Improve the per-pool stats Add pool-aggregated stats and fix underflows from some server versions. * tweaks
3817 lines
123 KiB
Go
3817 lines
123 KiB
Go
// Copyright (c) 2015-2026 MinIO, Inc.
|
|
//
|
|
// This file is part of MinIO Object Storage stack
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Affero General Public License as
|
|
// published by the Free Software Foundation, either version 3 of the
|
|
// License, or (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Affero General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
package madmin
|
|
|
|
import (
|
|
"cmp"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"maps"
|
|
"math"
|
|
"net/http"
|
|
"net/url"
|
|
"runtime/metrics"
|
|
"slices"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/prometheus/procfs"
|
|
"github.com/shirou/gopsutil/v4/cpu"
|
|
"github.com/shirou/gopsutil/v4/load"
|
|
"github.com/tinylib/msgp/msgp"
|
|
)
|
|
|
|
//go:generate go tool msgp -unexported -d clearomitted -d "tag json" -d "timezone utc" -d "maps binkeys" -file $GOFILE
|
|
|
|
//msgp:replace HealItemType with:string
|
|
|
|
// MetricType is a bitfield representation of different metric types.
|
|
type MetricType uint32
|
|
|
|
// MetricsNone indicates no metrics.
|
|
const MetricsNone MetricType = 0
|
|
|
|
const (
|
|
MetricsScanner MetricType = 1 << (iota)
|
|
MetricsDisk
|
|
MetricsOS
|
|
MetricsBatchJobs
|
|
MetricsSiteResync
|
|
MetricNet
|
|
MetricsMem
|
|
MetricsCPU
|
|
MetricsRPC
|
|
MetricsRuntime
|
|
MetricsAPI
|
|
MetricsReplication
|
|
MetricsProcess
|
|
MetricsHealing
|
|
MetricsBuckets
|
|
MetricsKMS
|
|
MetricsTablesAPI
|
|
|
|
// MetricsAll must be last.
|
|
// Enables all metrics.
|
|
MetricsAll = 1<<(iota) - 1
|
|
)
|
|
|
|
// Contains returns whether m contains all of x.
|
|
func (m MetricType) Contains(x MetricType) bool {
|
|
return m&x == x
|
|
}
|
|
|
|
// String returns a comma separated list of flags as string.
|
|
func (m MetricType) String() string {
|
|
var b strings.Builder
|
|
addIf := func(cond bool, str string) {
|
|
if cond {
|
|
if b.Len() > 0 {
|
|
b.WriteByte(',')
|
|
}
|
|
b.WriteString(str)
|
|
}
|
|
}
|
|
addIf(m.Contains(MetricsScanner), "Scanner")
|
|
addIf(m.Contains(MetricsDisk), "Disk")
|
|
addIf(m.Contains(MetricsOS), "OS")
|
|
addIf(m.Contains(MetricsBatchJobs), "BatchJobs")
|
|
addIf(m.Contains(MetricsSiteResync), "SiteResync")
|
|
addIf(m.Contains(MetricNet), "Net")
|
|
addIf(m.Contains(MetricsMem), "Mem")
|
|
addIf(m.Contains(MetricsCPU), "CPU")
|
|
addIf(m.Contains(MetricsRPC), "RPC")
|
|
addIf(m.Contains(MetricsRuntime), "Runtime")
|
|
addIf(m.Contains(MetricsAPI), "API")
|
|
addIf(m.Contains(MetricsReplication), "Replication")
|
|
addIf(m.Contains(MetricsProcess), "Process")
|
|
addIf(m.Contains(MetricsHealing), "Healing")
|
|
addIf(m.Contains(MetricsBuckets), "Buckets")
|
|
addIf(m.Contains(MetricsKMS), "KMS")
|
|
addIf(m.Contains(MetricsTablesAPI), "Tables API")
|
|
return b.String()
|
|
}
|
|
|
|
// MetricFlags is a bitfield representation of different metric flags.
|
|
type MetricFlags uint64
|
|
|
|
const (
|
|
MetricsDayStats MetricFlags = 1 << (iota) // Include daily statistics (24h, 15-min segments)
|
|
MetricsByHost // Aggregate metrics by host/node.
|
|
MetricsByDisk // Aggregate metrics by disk.
|
|
MetricsLegacyDiskIO // Add legacy disk IO metrics.
|
|
MetricsByDiskSet // Aggregate metrics by disk pool+set index.
|
|
MetricsSMART // Include S.M.A.R.T. disk health data.
|
|
MetricsHourStats // Include last-hour statistics (1h, 1-min segments)
|
|
MetricsTopWarehouses // Include top-25 metrics by warehouse
|
|
MetricsTopNamespaces // Include top-25 metrics by namespace
|
|
MetricsTopTables // Include top-25 tables
|
|
)
|
|
|
|
// Contains returns whether m contains all of x.
|
|
func (m MetricFlags) Contains(x MetricFlags) bool {
|
|
return m&x == x
|
|
}
|
|
|
|
// Add one or more flags to m.
|
|
func (m *MetricFlags) Add(x ...MetricFlags) {
|
|
for _, v := range x {
|
|
*m = *m | v
|
|
}
|
|
}
|
|
|
|
// String returns a comma separated list of flags as string.
|
|
func (m MetricFlags) String() string {
|
|
var b strings.Builder
|
|
addIf := func(cond bool, str string) {
|
|
if cond {
|
|
if b.Len() > 0 {
|
|
b.WriteByte(',')
|
|
}
|
|
b.WriteString(str)
|
|
}
|
|
}
|
|
addIf(m.Contains(MetricsDayStats), "DayStats")
|
|
addIf(m.Contains(MetricsByHost), "ByHost")
|
|
addIf(m.Contains(MetricsByDisk), "ByDisk")
|
|
addIf(m.Contains(MetricsLegacyDiskIO), "LegacyIO")
|
|
addIf(m.Contains(MetricsByDiskSet), "ByDiskSet")
|
|
addIf(m.Contains(MetricsSMART), "SMART")
|
|
addIf(m.Contains(MetricsHourStats), "HourStats")
|
|
return b.String()
|
|
}
|
|
|
|
// MetricsOptions are options provided to Metrics call.
|
|
type MetricsOptions struct {
|
|
Type MetricType // Return only these metric types. Several types can be combined using |. Leave at 0 to return all.
|
|
Flags MetricFlags // Flags to control returned metrics.
|
|
N int // Maximum number of samples to return. 0 will return endless stream.
|
|
Interval time.Duration // Interval between samples. Will be rounded up to 1s.
|
|
PoolIdx []int // Only include metrics for these pools. Leave empty for all.
|
|
Hosts []string // Only include specified hosts. Leave empty for all.
|
|
DrivePoolIdx []int // Only include metrics for these drive pools. Leave empty for all.
|
|
DriveSetIdx []int // Only include metrics for these drive sets (combine with PoolIdx if needed).
|
|
Disks []string // Include only specific disks. Leave empty for all.
|
|
Buckets []string // Include only specific buckets in bucket metrics. Leave empty for all.
|
|
ByJobID string
|
|
ByDepID string
|
|
|
|
// Alternative output merging.
|
|
// Populates maps of the same name in the result.
|
|
ByHost bool // Return individual metrics by host. Deprecated: use MetricsByHost instead.
|
|
ByDisk bool // Return individual metrics by disk. Deprecated: use MetricsByDisk instead.
|
|
}
|
|
|
|
// DriveSetPrefix will be used to select drives from specific sets.
|
|
const (
|
|
DriveSetPrefix = "::drive-set::"
|
|
DrivePoolPrefix = "::drive-pool::"
|
|
)
|
|
|
|
// Metrics makes an admin call to retrieve metrics.
|
|
// The provided function is called for each received entry.
|
|
func (adm *AdminClient) Metrics(ctx context.Context, o MetricsOptions, out func(RealtimeMetrics)) (err error) {
|
|
path := adminAPIPrefix + "/metrics"
|
|
q := make(url.Values)
|
|
q.Set("types", strconv.FormatUint(uint64(o.Type), 10))
|
|
q.Set("n", strconv.Itoa(o.N))
|
|
q.Set("interval", o.Interval.String())
|
|
q.Set("hosts", strings.Join(o.Hosts, ","))
|
|
if o.ByHost {
|
|
q.Set("by-host", "true") // Legacy flag
|
|
o.Flags.Add(MetricsByDisk)
|
|
}
|
|
for _, v := range o.DriveSetIdx {
|
|
o.Disks = append(o.Disks, fmt.Sprintf(DriveSetPrefix+"%d", v))
|
|
}
|
|
for _, v := range o.DrivePoolIdx {
|
|
o.Disks = append(o.Disks, fmt.Sprintf(DrivePoolPrefix+"%d", v))
|
|
}
|
|
|
|
q.Set("disks", strings.Join(o.Disks, ","))
|
|
if len(o.Buckets) > 0 {
|
|
q.Set("buckets", strings.Join(o.Buckets, ","))
|
|
}
|
|
if o.ByDisk {
|
|
q.Set("by-disk", "true") // Legacy flag
|
|
o.Flags.Add(MetricsByDisk)
|
|
}
|
|
if o.ByJobID != "" {
|
|
q.Set("by-jobID", o.ByJobID)
|
|
}
|
|
if o.ByDepID != "" {
|
|
q.Set("by-depID", o.ByDepID)
|
|
}
|
|
if len(o.PoolIdx) > 0 {
|
|
str := make([]string, len(o.PoolIdx))
|
|
for i, id := range o.PoolIdx {
|
|
str[i] = strconv.Itoa(id)
|
|
}
|
|
q.Set("pool-idx", strings.Join(str, ","))
|
|
}
|
|
q.Set("flags", strconv.FormatUint(uint64(o.Flags), 10))
|
|
|
|
resp, err := adm.executeMethod(ctx,
|
|
http.MethodGet, requestData{
|
|
customHeaders: map[string][]string{
|
|
"Accept": {"application/vnd.msgpack"},
|
|
},
|
|
relPath: path,
|
|
queryValues: q,
|
|
},
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return httpRespToErrorResponse(resp)
|
|
}
|
|
defer closeResponse(resp)
|
|
|
|
// Choose decoder based on content type
|
|
var decodeOne func(m *RealtimeMetrics) error
|
|
switch resp.Header.Get("Content-Type") {
|
|
case "application/vnd.msgpack":
|
|
dec := msgp.NewReader(resp.Body)
|
|
decodeOne = func(m *RealtimeMetrics) error {
|
|
return m.DecodeMsg(dec)
|
|
}
|
|
default:
|
|
dec := json.NewDecoder(resp.Body)
|
|
decodeOne = func(m *RealtimeMetrics) error {
|
|
return dec.Decode(m)
|
|
}
|
|
}
|
|
for {
|
|
var m RealtimeMetrics
|
|
err := decodeOne(&m)
|
|
if err != nil {
|
|
if errors.Is(err, io.EOF) {
|
|
err = io.ErrUnexpectedEOF
|
|
}
|
|
return err
|
|
}
|
|
if m.CollectedAt.IsZero() {
|
|
m.CollectedAt = time.Now()
|
|
}
|
|
out(m)
|
|
if m.Final {
|
|
break
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// RealtimeMetrics provides realtime metrics.
|
|
// This is intended to be expanded over time to cover more types.
|
|
type RealtimeMetrics struct {
|
|
// CollectedAt is the time these metrics were collected.
|
|
CollectedAt time.Time `json:"collected"`
|
|
|
|
// Error indicates an error occurred.
|
|
Errors []string `json:"errors,omitempty"`
|
|
|
|
// Hosts indicates the scanned hosts
|
|
Hosts []string `json:"hosts"`
|
|
|
|
// Aggregated contains aggregated metrics for all hosts
|
|
Aggregated Metrics `json:"aggregated"`
|
|
|
|
// ByHost contains metrics for each host if requested.
|
|
ByHost map[string]Metrics `json:"by_host,omitempty"`
|
|
|
|
// ByDisk contains metrics for each disk if requested.
|
|
ByDisk map[string]DiskMetric `json:"by_disk,omitempty"`
|
|
|
|
// ByDiskSet contains disk metrics aggregated by pool+set index.
|
|
ByDiskSet map[int]map[int]DiskMetric `json:"by_disk_set,omitempty"`
|
|
|
|
// Final indicates whether this is the final packet and the receiver can exit.
|
|
Final bool `json:"final"`
|
|
}
|
|
|
|
// Merge functionality:
|
|
//
|
|
// Overall rules: a.Merge(b)
|
|
//
|
|
// 1. All metrics must be accumulated and must be independent of order of merges.
|
|
// 2. If a field is not set in the other, it is not modified.
|
|
// 3. If a field is set in both, the value is merged.
|
|
// 4. Only a may be mutated.
|
|
// 5. 'a' can be the zero value.
|
|
|
|
// Merge will merge other into r.
|
|
func (r *RealtimeMetrics) Merge(other *RealtimeMetrics) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
if r.CollectedAt.Before(other.CollectedAt) {
|
|
r.CollectedAt = other.CollectedAt
|
|
}
|
|
|
|
if len(other.Errors) > 0 {
|
|
r.Errors = append(r.Errors, other.Errors...)
|
|
}
|
|
|
|
if r.ByHost == nil && len(other.ByHost) > 0 {
|
|
r.ByHost = make(map[string]Metrics, len(other.ByHost))
|
|
}
|
|
for host, metrics := range other.ByHost {
|
|
r.ByHost[host] = metrics
|
|
}
|
|
|
|
r.Hosts = append(r.Hosts, other.Hosts...)
|
|
r.Aggregated.Merge(&other.Aggregated)
|
|
sort.Strings(r.Hosts)
|
|
|
|
// Gather per disk metrics
|
|
if r.ByDisk == nil && len(other.ByDisk) > 0 {
|
|
r.ByDisk = make(map[string]DiskMetric, len(other.ByDisk))
|
|
}
|
|
for disk, metrics := range other.ByDisk {
|
|
r.ByDisk[disk] = metrics
|
|
}
|
|
if r.ByDiskSet == nil && len(other.ByDiskSet) > 0 {
|
|
r.ByDiskSet = make(map[int]map[int]DiskMetric, len(other.ByDisk))
|
|
}
|
|
for pIdx, pool := range other.ByDiskSet {
|
|
dstp := r.ByDiskSet[pIdx]
|
|
if dstp == nil {
|
|
dstp = make(map[int]DiskMetric, len(pool))
|
|
r.ByDiskSet[pIdx] = dstp
|
|
}
|
|
for sIdx, disks := range pool {
|
|
dsts := dstp[sIdx]
|
|
dsts.Merge(&disks)
|
|
dstp[sIdx] = dsts
|
|
}
|
|
}
|
|
}
|
|
|
|
// Metrics contains all metric types.
|
|
type Metrics struct {
|
|
Scanner *ScannerMetrics `json:"scanner,omitempty"`
|
|
Disk *DiskMetric `json:"disk,omitempty"`
|
|
OS *OSMetrics `json:"os,omitempty"`
|
|
BatchJobs *BatchJobMetrics `json:"batchJobs,omitempty"`
|
|
SiteResync *SiteResyncMetrics `json:"siteResync,omitempty"`
|
|
Net *NetMetrics `json:"net,omitempty"`
|
|
Mem *MemMetrics `json:"mem,omitempty"`
|
|
CPU *CPUMetrics `json:"cpu,omitempty"`
|
|
RPC *RPCMetrics `json:"rpc,omitempty"`
|
|
Go *RuntimeMetrics `json:"go,omitempty"`
|
|
API *APIMetrics `json:"api,omitempty"`
|
|
Replication *ReplicationMetrics `json:"replication,omitempty"`
|
|
Process *ProcessMetrics `json:"process,omitempty"`
|
|
Healing *HealingMetrics `json:"healing,omitempty"`
|
|
Buckets *BucketAPIMetrics `json:"buckets,omitempty"`
|
|
KMS *KMSRtMetrics `json:"kms,omitempty"`
|
|
TablesAPI *TableAPIMetrics `json:"tables_api,omitempty"`
|
|
}
|
|
|
|
// Merge other into r.
|
|
func (r *Metrics) Merge(other *Metrics) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
if r.Scanner == nil && other.Scanner != nil {
|
|
r.Scanner = &ScannerMetrics{}
|
|
}
|
|
r.Scanner.Merge(other.Scanner)
|
|
|
|
if r.Disk == nil && other.Disk != nil {
|
|
r.Disk = &DiskMetric{}
|
|
}
|
|
r.Disk.Merge(other.Disk)
|
|
|
|
if r.OS == nil && other.OS != nil {
|
|
r.OS = &OSMetrics{}
|
|
}
|
|
r.OS.Merge(other.OS)
|
|
if r.BatchJobs == nil && other.BatchJobs != nil {
|
|
r.BatchJobs = &BatchJobMetrics{}
|
|
}
|
|
r.BatchJobs.Merge(other.BatchJobs)
|
|
|
|
if r.SiteResync == nil && other.SiteResync != nil {
|
|
r.SiteResync = &SiteResyncMetrics{}
|
|
}
|
|
r.SiteResync.Merge(other.SiteResync)
|
|
|
|
if r.Net == nil && other.Net != nil {
|
|
r.Net = &NetMetrics{}
|
|
}
|
|
r.Net.Merge(other.Net)
|
|
if r.RPC == nil && other.RPC != nil {
|
|
r.RPC = &RPCMetrics{}
|
|
}
|
|
r.RPC.Merge(other.RPC)
|
|
if r.Go == nil && other.Go != nil {
|
|
r.Go = &RuntimeMetrics{}
|
|
}
|
|
r.Go.Merge(other.Go)
|
|
if r.API == nil && other.API != nil {
|
|
r.API = &APIMetrics{}
|
|
}
|
|
r.API.Merge(other.API)
|
|
if r.Replication == nil && other.Replication != nil {
|
|
r.Replication = &ReplicationMetrics{}
|
|
}
|
|
r.Replication.Merge(other.Replication)
|
|
if r.Mem == nil && other.Mem != nil {
|
|
r.Mem = &MemMetrics{}
|
|
}
|
|
r.Mem.Merge(other.Mem)
|
|
if r.CPU == nil && other.CPU != nil {
|
|
r.CPU = &CPUMetrics{}
|
|
}
|
|
r.CPU.Merge(other.CPU)
|
|
if r.Process == nil && other.Process != nil {
|
|
r.Process = &ProcessMetrics{}
|
|
}
|
|
r.Process.Merge(other.Process)
|
|
if r.Healing == nil && other.Healing != nil {
|
|
r.Healing = &HealingMetrics{}
|
|
}
|
|
r.Healing.Merge(other.Healing)
|
|
if r.Buckets == nil && other.Buckets != nil {
|
|
r.Buckets = &BucketAPIMetrics{}
|
|
}
|
|
r.Buckets.Merge(other.Buckets)
|
|
if other.KMS != nil {
|
|
if r.KMS == nil {
|
|
r.KMS = &KMSRtMetrics{}
|
|
}
|
|
r.KMS.Merge(other.KMS)
|
|
}
|
|
if other.TablesAPI != nil {
|
|
if r.TablesAPI == nil {
|
|
r.TablesAPI = &TableAPIMetrics{}
|
|
}
|
|
r.TablesAPI.Merge(other.TablesAPI)
|
|
}
|
|
}
|
|
|
|
// BucketILMStats reports the cumulative ILM action counters for a single
|
|
// bucket carried in a ScannerMetrics value.
|
|
type BucketILMStats struct {
|
|
Bucket string `json:"bucket,omitempty" msg:"bucket,omitempty"`
|
|
ActionCounters map[string]uint64 `json:"action_counters,omitempty" msg:"action_counters,omitempty"`
|
|
}
|
|
|
|
// Merge adds other.ActionCounters into b.ActionCounters. b.Bucket is preserved
|
|
// when set, otherwise adopted from other (even when other has no counters). A
|
|
// nil other is a no-op.
|
|
func (b *BucketILMStats) Merge(other *BucketILMStats) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
if b.Bucket == "" {
|
|
b.Bucket = other.Bucket
|
|
}
|
|
if len(other.ActionCounters) == 0 {
|
|
return
|
|
}
|
|
if b.ActionCounters == nil {
|
|
b.ActionCounters = make(map[string]uint64, len(other.ActionCounters))
|
|
}
|
|
for action, n := range other.ActionCounters {
|
|
b.ActionCounters[action] += n
|
|
}
|
|
}
|
|
|
|
// ScannerMetrics contains scanner information.
|
|
type ScannerMetrics struct {
|
|
// Time these metrics were collected
|
|
CollectedAt time.Time `json:"collected"`
|
|
|
|
// Number of buckets currently scanning
|
|
OngoingBuckets int `json:"ongoing_buckets"`
|
|
|
|
// Stats per bucket, a map between bucket name and scan stats in all erasure sets
|
|
PerBucketStats map[string][]BucketScanInfo `json:"per_bucket_stats,omitempty"`
|
|
|
|
// Number of accumulated operations by type since server restart.
|
|
LifeTimeOps map[string]uint64 `json:"life_time_ops,omitempty"`
|
|
|
|
// Number of accumulated ILM operations by type since server restart.
|
|
LifeTimeILM map[string]uint64 `json:"ilm_ops,omitempty"`
|
|
|
|
// BucketLifeTimeILM reports cumulative ILM action counters per bucket,
|
|
// keyed by bucket name. Populated only when a specific bucket is
|
|
// requested; nil otherwise.
|
|
BucketLifeTimeILM map[string]*BucketILMStats `json:"bucket_ilm_stats,omitempty"`
|
|
|
|
// Last minute operation statistics.
|
|
LastMinute struct {
|
|
// Scanner actions.
|
|
Actions map[string]TimedAction `json:"actions,omitempty"`
|
|
// ILM actions.
|
|
ILM map[string]TimedAction `json:"ilm,omitempty"`
|
|
} `json:"last_minute"`
|
|
|
|
// LastDay operation statistics.
|
|
LastDay map[string]SegmentedActions `json:"last_day,omitempty"`
|
|
|
|
// Currently active path(s) being scanned.
|
|
ActivePaths []string `json:"active,omitempty"`
|
|
|
|
// ExcessivePrefixes lists prefixes marked as having excessive sub-entries
|
|
// within the last 24 hours.
|
|
ExcessivePrefixes []string `json:"excessive,omitempty"`
|
|
|
|
// ExcessiveVersionObjects lists objects that have exceeded the version
|
|
// count or cumulative size threshold within the last 24 hours.
|
|
// Capped at 100 entries per cross-node merge; see DiscardedExcessEntries.
|
|
ExcessiveVersionObjects []string `json:"excessive_versions,omitempty"`
|
|
|
|
// DiscardedExcessEntries counts entries dropped beyond the 100-entry cap
|
|
// during cross-node merge. This counter is not deduplicated.
|
|
DiscardedExcessEntries uint64 `json:"discarded_excess_entries,omitempty"`
|
|
|
|
// Number of queued ILM expiry tasks.
|
|
ILMExpiryPendingTasks int `json:"ilm_expiry_pending_tasks,omitempty"`
|
|
// ILMExpiryTasksServiced tracks the last-minute latency and count of ILM expiry
|
|
// tasks that have been serviced, measured from queue time to completion.
|
|
ILMExpiryTasksServiced TimedAction `json:"ilm_expiry_tasks_cleanup"`
|
|
|
|
// QueuedForExpiry holds the most recently queued expiry objects
|
|
QueuedForExpiry []ExpiryObject `json:"queued_for_expiry,omitempty"`
|
|
}
|
|
|
|
// ExpiryObject contains information about an object recently queued for ILM expiry.
|
|
type ExpiryObject struct {
|
|
Bucket string `json:"bucket"`
|
|
Object string `json:"object"`
|
|
Versions int `json:"versions"`
|
|
QueuedAt time.Time `json:"queued_at"`
|
|
}
|
|
|
|
// Merge combines two lists of expiry objects into a single sorted list
|
|
// preserving order (newest first), the out is limited to max 25 objects.
|
|
func Merge(a, b []ExpiryObject) []ExpiryObject {
|
|
a = append(a, b...)
|
|
slices.SortFunc(a, func(a, b ExpiryObject) int {
|
|
res := b.QueuedAt.Compare(a.QueuedAt)
|
|
if res != 0 {
|
|
return res
|
|
}
|
|
res = cmp.Compare(a.Bucket, b.Bucket)
|
|
if res != 0 {
|
|
return res
|
|
}
|
|
return cmp.Compare(a.Object, b.Object)
|
|
})
|
|
return a[:min(len(a), 25)]
|
|
}
|
|
|
|
// SegmentedActions are time segmented scanner activity.
|
|
type SegmentedActions = Segmented[TimedAction, *TimedAction]
|
|
|
|
// Merge other into 's'.
|
|
func (s *ScannerMetrics) Merge(other *ScannerMetrics) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
|
|
if s.CollectedAt.Before(other.CollectedAt) {
|
|
// Use latest timestamp
|
|
s.CollectedAt = other.CollectedAt
|
|
}
|
|
|
|
if s.OngoingBuckets < other.OngoingBuckets {
|
|
s.OngoingBuckets = other.OngoingBuckets
|
|
}
|
|
|
|
if s.PerBucketStats == nil {
|
|
s.PerBucketStats = make(map[string][]BucketScanInfo)
|
|
}
|
|
for bucket, otherSt := range other.PerBucketStats {
|
|
if len(otherSt) == 0 {
|
|
continue
|
|
}
|
|
_, ok := s.PerBucketStats[bucket]
|
|
if !ok {
|
|
s.PerBucketStats[bucket] = otherSt
|
|
}
|
|
}
|
|
|
|
// Regular ops
|
|
if len(other.LifeTimeOps) > 0 && s.LifeTimeOps == nil {
|
|
s.LifeTimeOps = make(map[string]uint64, len(other.LifeTimeOps))
|
|
}
|
|
for k, v := range other.LifeTimeOps {
|
|
total := s.LifeTimeOps[k] + v
|
|
s.LifeTimeOps[k] = total
|
|
}
|
|
if s.LastMinute.Actions == nil && len(other.LastMinute.Actions) > 0 {
|
|
s.LastMinute.Actions = make(map[string]TimedAction, len(other.LastMinute.Actions))
|
|
}
|
|
for k, v := range other.LastMinute.Actions {
|
|
total := s.LastMinute.Actions[k]
|
|
total.Merge(v)
|
|
s.LastMinute.Actions[k] = total
|
|
}
|
|
if s.LastDay == nil && len(other.LastDay) > 0 {
|
|
s.LastDay = make(map[string]SegmentedActions, len(other.LastDay))
|
|
}
|
|
for k, v := range other.LastDay {
|
|
total := s.LastDay[k]
|
|
total.Add(&v)
|
|
s.LastDay[k] = total
|
|
}
|
|
|
|
// ILM
|
|
if len(other.LifeTimeILM) > 0 && s.LifeTimeILM == nil {
|
|
s.LifeTimeILM = make(map[string]uint64, len(other.LifeTimeILM))
|
|
}
|
|
for k, v := range other.LifeTimeILM {
|
|
total := s.LifeTimeILM[k] + v
|
|
s.LifeTimeILM[k] = total
|
|
}
|
|
for bucket, otherStats := range other.BucketLifeTimeILM {
|
|
if s.BucketLifeTimeILM == nil {
|
|
s.BucketLifeTimeILM = make(map[string]*BucketILMStats, len(other.BucketLifeTimeILM))
|
|
}
|
|
dst, ok := s.BucketLifeTimeILM[bucket]
|
|
if !ok {
|
|
dst = &BucketILMStats{Bucket: bucket}
|
|
s.BucketLifeTimeILM[bucket] = dst
|
|
}
|
|
dst.Merge(otherStats)
|
|
}
|
|
if s.LastMinute.ILM == nil && len(other.LastMinute.ILM) > 0 {
|
|
s.LastMinute.ILM = make(map[string]TimedAction, len(other.LastMinute.ILM))
|
|
}
|
|
for k, v := range other.LastMinute.ILM {
|
|
total := s.LastMinute.ILM[k]
|
|
total.Merge(v)
|
|
s.LastMinute.ILM[k] = total
|
|
}
|
|
s.ActivePaths = append(s.ActivePaths, other.ActivePaths...)
|
|
sort.Strings(s.ActivePaths)
|
|
|
|
if len(other.ExcessivePrefixes) > 0 {
|
|
merged := make(map[string]struct{}, len(s.ExcessivePrefixes)+len(other.ExcessivePrefixes))
|
|
for _, prefix := range s.ExcessivePrefixes {
|
|
merged[prefix] = struct{}{}
|
|
}
|
|
for _, prefix := range other.ExcessivePrefixes {
|
|
merged[prefix] = struct{}{}
|
|
}
|
|
s.ExcessivePrefixes = make([]string, 0, len(merged))
|
|
for prefix := range merged {
|
|
s.ExcessivePrefixes = append(s.ExcessivePrefixes, prefix)
|
|
}
|
|
sort.Strings(s.ExcessivePrefixes)
|
|
}
|
|
|
|
if len(other.ExcessiveVersionObjects) > 0 {
|
|
const maxExcessEntries = 100
|
|
seen := make(map[string]struct{}, len(s.ExcessiveVersionObjects)+len(other.ExcessiveVersionObjects))
|
|
for _, v := range s.ExcessiveVersionObjects {
|
|
seen[v] = struct{}{}
|
|
}
|
|
for _, v := range other.ExcessiveVersionObjects {
|
|
seen[v] = struct{}{}
|
|
}
|
|
keys := make([]string, 0, len(seen))
|
|
for k := range seen {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
if len(keys) > maxExcessEntries {
|
|
s.DiscardedExcessEntries += uint64(len(keys) - maxExcessEntries)
|
|
keys = keys[:maxExcessEntries]
|
|
}
|
|
s.ExcessiveVersionObjects = keys
|
|
}
|
|
s.DiscardedExcessEntries += other.DiscardedExcessEntries
|
|
|
|
s.ILMExpiryPendingTasks += other.ILMExpiryPendingTasks
|
|
s.ILMExpiryTasksServiced.Merge(other.ILMExpiryTasksServiced)
|
|
if len(other.QueuedForExpiry) > 0 {
|
|
s.QueuedForExpiry = Merge(s.QueuedForExpiry, other.QueuedForExpiry)
|
|
}
|
|
}
|
|
|
|
// DiskIOStats contains IO stats of a single drive
|
|
type DiskIOStats struct {
|
|
N int `json:"n,omitempty"`
|
|
// WithIOStats is the subset of N whose kernel sysfs iostat
|
|
// (`/sys/dev/block/.../stat`) was readable; consumers gate I/O UI on it.
|
|
WithIOStats int `json:"with_iostats,omitempty"`
|
|
ReadIOs uint64 `json:"read_ios,omitempty"`
|
|
ReadMerges uint64 `json:"read_merges,omitempty"`
|
|
ReadSectors uint64 `json:"read_sectors,omitempty"`
|
|
ReadTicks uint64 `json:"read_ticks,omitempty"`
|
|
WriteIOs uint64 `json:"write_ios,omitempty"`
|
|
WriteMerges uint64 `json:"write_merges,omitempty"`
|
|
WriteSectors uint64 `json:"write_sectors,omitempty"`
|
|
WriteTicks uint64 `json:"write_ticks,omitempty"`
|
|
CurrentIOs uint64 `json:"current_ios,omitempty"`
|
|
TotalTicks uint64 `json:"total_ticks,omitempty"`
|
|
ReqTicks uint64 `json:"req_ticks,omitempty"`
|
|
DiscardIOs uint64 `json:"discard_ios,omitempty"`
|
|
DiscardMerges uint64 `json:"discard_merges,omitempty"`
|
|
DiscardSectors uint64 `json:"discard_sectors,omitempty"`
|
|
DiscardTicks uint64 `json:"discard_ticks,omitempty"`
|
|
FlushIOs uint64 `json:"flush_ios,omitempty"`
|
|
FlushTicks uint64 `json:"flush_ticks,omitempty"`
|
|
BitrotDetected uint64 `json:"bitrot_detected,omitempty"`
|
|
BitrotHealed uint64 `json:"bitrot_healed,omitempty"`
|
|
}
|
|
|
|
// DiskIOStatsLegacy mirrors DiskIOStats field-for-field so direct Go type
|
|
// conversions (used in DiskMetric.Merge) stay valid; mirror new fields too.
|
|
type DiskIOStatsLegacy struct {
|
|
N int `json:"n,omitempty"`
|
|
WithIOStats int `json:"with_iostats,omitempty"`
|
|
ReadIOs uint64 `json:"read_ios,omitempty"`
|
|
ReadMerges uint64 `json:"read_merges,omitempty"`
|
|
ReadSectors uint64 `json:"read_sectors,omitempty"`
|
|
ReadTicks uint64 `json:"read_ticks,omitempty"`
|
|
WriteIOs uint64 `json:"write_ios,omitempty"`
|
|
WriteMerges uint64 `json:"write_merges,omitempty"`
|
|
WriteSectors uint64 `json:"wrte_sectors,omitempty"` // note "spelling"
|
|
WriteTicks uint64 `json:"write_ticks,omitempty"`
|
|
CurrentIOs uint64 `json:"current_ios,omitempty"`
|
|
TotalTicks uint64 `json:"total_ticks,omitempty"`
|
|
ReqTicks uint64 `json:"req_ticks,omitempty"`
|
|
DiscardIOs uint64 `json:"discard_ios,omitempty"`
|
|
DiscardMerges uint64 `json:"discard_merges,omitempty"`
|
|
DiscardSectors uint64 `json:"discard_secotrs,omitempty"` // note "spelling"
|
|
DiscardTicks uint64 `json:"discard_ticks,omitempty"`
|
|
FlushIOs uint64 `json:"flush_ios,omitempty"`
|
|
FlushTicks uint64 `json:"flush_ticks,omitempty"`
|
|
BitrotDetected uint64 `json:"bitrot_detected,omitempty"`
|
|
BitrotHealed uint64 `json:"bitrot_healed,omitempty"`
|
|
}
|
|
|
|
// Add 'other' to 'd'.
|
|
func (d *DiskIOStats) Add(other *DiskIOStats) {
|
|
if other == nil || other.overflowed() {
|
|
// Discard segments carrying a uint64 underflow artifact (see overflowed)
|
|
// instead of polluting the aggregate.
|
|
return
|
|
}
|
|
if d.overflowed() {
|
|
// A corrupt receiver would otherwise drag its uint64 underflow artifact
|
|
// into every subsequent sum; reset it and restart the aggregate from the
|
|
// valid other value.
|
|
*d = DiskIOStats{}
|
|
}
|
|
d.N += other.N
|
|
d.WithIOStats += other.WithIOStats
|
|
d.ReadIOs += other.ReadIOs
|
|
d.ReadMerges += other.ReadMerges
|
|
d.ReadSectors += other.ReadSectors
|
|
d.ReadTicks += other.ReadTicks
|
|
d.WriteIOs += other.WriteIOs
|
|
d.WriteMerges += other.WriteMerges
|
|
d.WriteSectors += other.WriteSectors
|
|
d.WriteTicks += other.WriteTicks
|
|
d.CurrentIOs += other.CurrentIOs
|
|
d.TotalTicks += other.TotalTicks
|
|
d.ReqTicks += other.ReqTicks
|
|
d.DiscardIOs += other.DiscardIOs
|
|
d.DiscardMerges += other.DiscardMerges
|
|
d.DiscardSectors += other.DiscardSectors
|
|
d.DiscardTicks += other.DiscardTicks
|
|
d.FlushIOs += other.FlushIOs
|
|
d.FlushTicks += other.FlushTicks
|
|
d.BitrotDetected += other.BitrotDetected
|
|
d.BitrotHealed += other.BitrotHealed
|
|
}
|
|
|
|
// overflowed reports whether any counter has its high bit set (value >
|
|
// math.MaxInt64) — the signature of a uint64 underflow. Kernel iostat deltas
|
|
// underflow to ~MaxUint64 on counter resets (reboot, drive hot-swap, wrap),
|
|
// which is nonsensical as an IO count, so such segments are discarded on merge.
|
|
func (d *DiskIOStats) overflowed() bool {
|
|
return (d.ReadIOs | d.ReadMerges | d.ReadSectors | d.ReadTicks |
|
|
d.WriteIOs | d.WriteMerges | d.WriteSectors | d.WriteTicks |
|
|
d.CurrentIOs | d.TotalTicks | d.ReqTicks |
|
|
d.DiscardIOs | d.DiscardMerges | d.DiscardSectors | d.DiscardTicks |
|
|
d.FlushIOs | d.FlushTicks | d.BitrotDetected | d.BitrotHealed) > math.MaxInt64
|
|
}
|
|
|
|
// discardOverflowedSegments zeroes IO segments carrying a uint64 underflow
|
|
// artifact (see DiskIOStats.overflowed) so they do not survive a merge.
|
|
func discardOverflowedSegments(segs []DiskIOStats) {
|
|
for i := range segs {
|
|
if segs[i].overflowed() {
|
|
segs[i] = DiskIOStats{}
|
|
}
|
|
}
|
|
}
|
|
|
|
type (
|
|
SegmentedDiskActions = Segmented[DiskAction, *DiskAction]
|
|
SegmentedDiskIO = Segmented[DiskIOStats, *DiskIOStats]
|
|
)
|
|
|
|
// DiskMetric contains metrics for one or more disks.
|
|
type DiskMetric struct {
|
|
// Time these metrics were collected
|
|
CollectedAt time.Time `json:"collected"`
|
|
|
|
// Number of disks
|
|
NDisks int `json:"n_disks"`
|
|
|
|
// DiskIdx will be populated if all disks in the metrics have the same drive index.
|
|
DiskIdx *int `json:"disk_idx,omitempty"`
|
|
|
|
// SetIdx will be populated if all disks in the metrics are part of the same set.
|
|
SetIdx *int `json:"set_idx,omitempty"`
|
|
|
|
// PoolIdx will be populated if all disks in the metrics are part of the same pool.
|
|
PoolIdx *int `json:"pool_idx,omitempty"`
|
|
|
|
// Disk states for non-ok disks.
|
|
// See madmin.DriveState for possible values.
|
|
State map[string]int `json:"state,omitempty"`
|
|
|
|
// Offline disks
|
|
Offline int `json:"offline,omitempty"`
|
|
|
|
// Hanging - drives hanging.
|
|
Hanging int `json:"waiting,omitempty"`
|
|
|
|
// Healing disks
|
|
// Deprecated, will be removed in later releases
|
|
Healing int `json:"healing,omitempty"`
|
|
|
|
// HealingInfo gives us a high level overview of the drives healing state
|
|
HealingInfo *DriveHealInfo `json:"healingInfo,omitempty"`
|
|
|
|
// Cache stats if enabled.
|
|
Cache *CacheStats `json:"cache,omitempty"`
|
|
|
|
// Space info.
|
|
Space DriveSpaceInfo `json:"space"`
|
|
|
|
// Number of accumulated operations by type.
|
|
LifetimeOps map[string]DiskAction `json:"lifetime_ops,omitempty"`
|
|
|
|
// Last minute statistics.
|
|
LastMinute map[string]DiskAction `json:"last_minute,omitempty"`
|
|
|
|
// LastDaySegmented contains the segmented metrics for the last day.
|
|
LastDaySegmented map[string]SegmentedDiskActions `json:"last_day,omitempty"`
|
|
|
|
// LastHourSegmented contains the segmented metrics for the last hour.
|
|
LastHourSegmented map[string]SegmentedDiskActions `json:"last_hour,omitempty"`
|
|
|
|
// IO stats.
|
|
// Deprecated: use io_min, io_day instead.
|
|
IOStats *DiskIOStatsLegacy `json:"iostats,omitempty"`
|
|
|
|
// Rolling window last minute IO stats.
|
|
IOStatsMinute DiskIOStats `json:"io_min"`
|
|
|
|
// Rolling window daily IO stats (15-minute segments).
|
|
IOStatsDay SegmentedDiskIO `json:"io_day"`
|
|
|
|
// Rolling window hourly IO stats (1-minute segments).
|
|
IOStatsHour SegmentedDiskIO `json:"io_hour"`
|
|
|
|
// SMART health data for the disk.
|
|
SMART *SMARTInfo `json:"smart,omitempty"`
|
|
|
|
// Filesystem type (e.g. "xfs", "ext4" and count).
|
|
FSType map[string]int `json:"fsType,omitempty"`
|
|
}
|
|
|
|
type DriveHealInfo struct {
|
|
ItemsHealed uint64 `json:"itemsHealed"`
|
|
ItemsFailed uint64 `json:"itemsFailed"`
|
|
HealID string `json:"healID"`
|
|
Finished bool `json:"finished"`
|
|
Started time.Time `json:"started"`
|
|
Updated time.Time `json:"updated"`
|
|
}
|
|
|
|
// DriveSpaceInfo is the space info of one or more drives.
|
|
type DriveSpaceInfo struct {
|
|
N int `json:"n"`
|
|
Free TotalMinMaxUint64 `json:"free"`
|
|
Used TotalMinMaxUint64 `json:"used"`
|
|
UsedInodes TotalMinMaxUint64 `json:"used_inodes"`
|
|
FreeInodes TotalMinMaxUint64 `json:"free_inodes"`
|
|
}
|
|
|
|
func (d *DriveSpaceInfo) Merge(other DriveSpaceInfo) {
|
|
d.N += other.N
|
|
d.Free.Merge(other.Free, d.N)
|
|
d.Used.Merge(other.Used, d.N)
|
|
d.UsedInodes.Merge(other.UsedInodes, d.N)
|
|
d.FreeInodes.Merge(other.FreeInodes, d.N)
|
|
}
|
|
|
|
//msgp:tuple TotalMinMaxUint64
|
|
type TotalMinMaxUint64 struct {
|
|
Total uint64 `json:"total"`
|
|
Min uint64 `json:"min"`
|
|
Max uint64 `json:"max"`
|
|
}
|
|
|
|
func (t *TotalMinMaxUint64) SetAll(v uint64) {
|
|
t.Total = v
|
|
t.Min = v
|
|
t.Max = v
|
|
}
|
|
|
|
// Merge 'other' into 't', assuming both are set.
|
|
func (t *TotalMinMaxUint64) Merge(other TotalMinMaxUint64, tCnt int) {
|
|
t.Total += other.Total
|
|
if tCnt == 0 || t.Min > other.Min {
|
|
t.Min = other.Min
|
|
}
|
|
t.Max = max(t.Max, other.Max)
|
|
}
|
|
|
|
// Merge other into 's'.
|
|
func (d *DiskMetric) Merge(other *DiskMetric) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
if d.NDisks == 0 {
|
|
*d = *other
|
|
// *d = *other aliases the IO segment slices; take independent copies and
|
|
// drop overflowed segments so a corrupt source segment neither survives
|
|
// the merge nor is mutated by it.
|
|
d.IOStatsDay.Segments = slices.Clone(other.IOStatsDay.Segments)
|
|
discardOverflowedSegments(d.IOStatsDay.Segments)
|
|
d.IOStatsHour.Segments = slices.Clone(other.IOStatsHour.Segments)
|
|
discardOverflowedSegments(d.IOStatsHour.Segments)
|
|
d.State = maps.Clone(other.State)
|
|
d.FSType = maps.Clone(other.FSType)
|
|
d.LifetimeOps = maps.Clone(other.LifetimeOps)
|
|
d.LastMinute = maps.Clone(other.LastMinute)
|
|
if other.LastDaySegmented != nil {
|
|
d.LastDaySegmented = make(map[string]SegmentedDiskActions, len(other.LastDaySegmented))
|
|
for k, v := range other.LastDaySegmented {
|
|
v.Segments = slices.Clone(v.Segments)
|
|
d.LastDaySegmented[k] = v
|
|
}
|
|
}
|
|
if other.Cache != nil {
|
|
c := *other.Cache
|
|
d.Cache = &c
|
|
}
|
|
if other.SMART != nil {
|
|
s := *other.SMART
|
|
s.Status = maps.Clone(other.SMART.Status)
|
|
if other.SMART.NVMe != nil {
|
|
nvme := *other.SMART.NVMe
|
|
s.NVMe = &nvme
|
|
}
|
|
if other.SMART.SATA != nil {
|
|
sata := *other.SMART.SATA
|
|
s.SATA = &sata
|
|
}
|
|
d.SMART = &s
|
|
}
|
|
if other.LastHourSegmented != nil {
|
|
d.LastHourSegmented = make(map[string]SegmentedDiskActions, len(other.LastHourSegmented))
|
|
for k, v := range other.LastHourSegmented {
|
|
v.Segments = slices.Clone(v.Segments)
|
|
d.LastHourSegmented[k] = v
|
|
}
|
|
}
|
|
return
|
|
}
|
|
if d.CollectedAt.Before(other.CollectedAt) {
|
|
// Use latest timestamp
|
|
d.CollectedAt = other.CollectedAt
|
|
}
|
|
// PoolIdx and SetIdx must match for all disks in the metrics.
|
|
if d.PoolIdx == nil && d.NDisks == 0 && other.PoolIdx != nil {
|
|
d.PoolIdx = other.PoolIdx
|
|
} else if other.PoolIdx == nil || d.PoolIdx != nil && other.PoolIdx != nil && *d.PoolIdx != *other.PoolIdx {
|
|
d.PoolIdx = nil
|
|
}
|
|
if d.SetIdx == nil && d.NDisks == 0 && other.SetIdx != nil {
|
|
d.SetIdx = other.SetIdx
|
|
} else if other.SetIdx == nil || d.SetIdx != nil && other.SetIdx != nil && *d.SetIdx != *other.SetIdx || d.PoolIdx == nil {
|
|
d.SetIdx = nil
|
|
}
|
|
if d.DiskIdx == nil && d.NDisks == 0 && other.DiskIdx != nil {
|
|
d.DiskIdx = other.DiskIdx
|
|
} else if other.DiskIdx == nil || d.DiskIdx != nil && other.DiskIdx != nil && *d.DiskIdx != *other.DiskIdx || d.SetIdx == nil {
|
|
d.DiskIdx = nil
|
|
}
|
|
if len(other.State) > 0 {
|
|
if d.State == nil {
|
|
d.State = make(map[string]int, len(other.State))
|
|
}
|
|
for k, v := range other.State {
|
|
d.State[k] = d.State[k] + v
|
|
}
|
|
}
|
|
if len(other.FSType) > 0 {
|
|
if d.FSType == nil {
|
|
d.FSType = make(map[string]int, len(other.FSType))
|
|
}
|
|
for k, v := range other.FSType {
|
|
d.FSType[k] += v
|
|
}
|
|
}
|
|
d.NDisks += other.NDisks
|
|
d.Offline += other.Offline
|
|
d.Healing += other.Healing
|
|
d.Hanging += other.Hanging
|
|
if other.Cache != nil {
|
|
if d.Cache == nil {
|
|
c := *other.Cache
|
|
d.Cache = &c
|
|
} else {
|
|
d.Cache.Merge(other.Cache)
|
|
}
|
|
}
|
|
d.Space.Merge(other.Space)
|
|
|
|
if len(other.LifetimeOps) > 0 && d.LifetimeOps == nil {
|
|
d.LifetimeOps = make(map[string]DiskAction, len(other.LifetimeOps))
|
|
}
|
|
for k, v := range other.LifetimeOps {
|
|
t := d.LifetimeOps[k]
|
|
t.Add(&v)
|
|
d.LifetimeOps[k] = t
|
|
}
|
|
|
|
if d.LastMinute == nil && len(other.LastMinute) > 0 {
|
|
d.LastMinute = make(map[string]DiskAction, len(other.LastMinute))
|
|
}
|
|
for k, v := range other.LastMinute {
|
|
t := d.LastMinute[k]
|
|
t.Add(&v)
|
|
d.LastMinute[k] = t
|
|
}
|
|
|
|
if len(other.LastDaySegmented) > 0 && d.LastDaySegmented == nil {
|
|
d.LastDaySegmented = make(map[string]SegmentedDiskActions, len(other.LastDaySegmented))
|
|
}
|
|
for k, v := range other.LastDaySegmented {
|
|
t := d.LastDaySegmented[k]
|
|
t.Add(&v)
|
|
d.LastDaySegmented[k] = t
|
|
}
|
|
|
|
if len(other.LastHourSegmented) > 0 && d.LastHourSegmented == nil {
|
|
d.LastHourSegmented = make(map[string]SegmentedDiskActions, len(other.LastHourSegmented))
|
|
}
|
|
for k, v := range other.LastHourSegmented {
|
|
t := d.LastHourSegmented[k]
|
|
t.Add(&v)
|
|
d.LastHourSegmented[k] = t
|
|
}
|
|
|
|
if other.IOStats != nil {
|
|
if d.IOStats == nil {
|
|
d.IOStats = new(DiskIOStatsLegacy)
|
|
}
|
|
a, b := DiskIOStats(*d.IOStats), DiskIOStats(*other.IOStats)
|
|
a.Add(&b)
|
|
c := DiskIOStatsLegacy(a)
|
|
d.IOStats = &c
|
|
}
|
|
d.IOStatsMinute.Add(&other.IOStatsMinute)
|
|
d.IOStatsDay.Add(&other.IOStatsDay)
|
|
discardOverflowedSegments(d.IOStatsDay.Segments)
|
|
d.IOStatsHour.Add(&other.IOStatsHour)
|
|
discardOverflowedSegments(d.IOStatsHour.Segments)
|
|
// Merge SMART data
|
|
if other.SMART != nil {
|
|
if d.SMART == nil {
|
|
d.SMART = &SMARTInfo{}
|
|
}
|
|
d.SMART.Merge(other.SMART)
|
|
}
|
|
}
|
|
|
|
// LifetimeTotal returns the accumulated Disk metrics for all operations
|
|
func (d DiskMetric) LifetimeTotal() DiskAction {
|
|
var res DiskAction
|
|
for _, s := range d.LifetimeOps {
|
|
res.Add(&s)
|
|
}
|
|
return res
|
|
}
|
|
|
|
// SensorMetrics aggregated sensor metrics for a single sensor key
|
|
type SensorMetrics struct {
|
|
MinTemp float64 `json:"min_temp"` // Minimum temperature seen
|
|
MaxTemp float64 `json:"max_temp"` // Maximum temperature seen
|
|
TotalTemp float64 `json:"total_temp"` // Total temperature for averaging
|
|
Count int `json:"count"` // Number of readings
|
|
ExceedsCritical int `json:"exceeds_critical,omitempty"` // Count of readings exceeding critical threshold
|
|
}
|
|
|
|
// OSMetrics contains metrics for OS operations.
|
|
type OSMetrics struct {
|
|
// Time these metrics were collected
|
|
CollectedAt time.Time `json:"collected"`
|
|
|
|
// Number of accumulated operations by type since server restart.
|
|
LifeTimeOps map[string]uint64 `json:"life_time_ops,omitempty"`
|
|
|
|
// Last minute statistics.
|
|
LastMinute struct {
|
|
Operations map[string]TimedAction `json:"operations,omitempty"`
|
|
} `json:"last_minute"`
|
|
|
|
// LastDay operation statistics.
|
|
LastDay map[string]SegmentedActions `json:"last_day,omitempty"`
|
|
|
|
// Aggregated temperature sensor metrics by sensor key
|
|
Sensors map[string]SensorMetrics `json:"sensors,omitempty"`
|
|
}
|
|
|
|
// Merge other into 'o'.
|
|
func (o *OSMetrics) Merge(other *OSMetrics) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
if o.CollectedAt.Before(other.CollectedAt) {
|
|
// Use latest timestamp
|
|
o.CollectedAt = other.CollectedAt
|
|
}
|
|
|
|
if len(other.LifeTimeOps) > 0 && o.LifeTimeOps == nil {
|
|
o.LifeTimeOps = make(map[string]uint64, len(other.LifeTimeOps))
|
|
}
|
|
for k, v := range other.LifeTimeOps {
|
|
total := o.LifeTimeOps[k] + v
|
|
o.LifeTimeOps[k] = total
|
|
}
|
|
|
|
if o.LastMinute.Operations == nil && len(other.LastMinute.Operations) > 0 {
|
|
o.LastMinute.Operations = make(map[string]TimedAction, len(other.LastMinute.Operations))
|
|
}
|
|
for k, v := range other.LastMinute.Operations {
|
|
total := o.LastMinute.Operations[k]
|
|
total.Merge(v)
|
|
o.LastMinute.Operations[k] = total
|
|
}
|
|
|
|
// Merge sensor metrics
|
|
if len(other.Sensors) > 0 {
|
|
if o.Sensors == nil {
|
|
o.Sensors = make(map[string]SensorMetrics)
|
|
}
|
|
for key, otherSensor := range other.Sensors {
|
|
existing := o.Sensors[key]
|
|
// Handle min/max
|
|
if existing.Count == 0 {
|
|
// First data for this sensor
|
|
existing.MinTemp = otherSensor.MinTemp
|
|
existing.MaxTemp = otherSensor.MaxTemp
|
|
} else {
|
|
if otherSensor.MinTemp < existing.MinTemp {
|
|
existing.MinTemp = otherSensor.MinTemp
|
|
}
|
|
if otherSensor.MaxTemp > existing.MaxTemp {
|
|
existing.MaxTemp = otherSensor.MaxTemp
|
|
}
|
|
}
|
|
// Accumulate totals
|
|
existing.TotalTemp += otherSensor.TotalTemp
|
|
existing.Count += otherSensor.Count
|
|
existing.ExceedsCritical += otherSensor.ExceedsCritical
|
|
o.Sensors[key] = existing
|
|
}
|
|
}
|
|
|
|
// Merge LastDay statistics
|
|
if len(other.LastDay) > 0 {
|
|
if o.LastDay == nil {
|
|
o.LastDay = make(map[string]SegmentedActions, len(other.LastDay))
|
|
}
|
|
for k, v := range other.LastDay {
|
|
total := o.LastDay[k]
|
|
total.Add(&v)
|
|
o.LastDay[k] = total
|
|
}
|
|
}
|
|
}
|
|
|
|
// BatchJobMetrics contains metrics for batch operations
|
|
type BatchJobMetrics struct {
|
|
// Time these metrics were collected
|
|
CollectedAt time.Time `json:"collected"`
|
|
|
|
// Jobs by ID.
|
|
Jobs map[string]JobMetric
|
|
}
|
|
|
|
type JobMetric struct {
|
|
JobID string `json:"jobID"`
|
|
JobType string `json:"jobType"`
|
|
StartTime time.Time `json:"startTime"`
|
|
LastUpdate time.Time `json:"lastUpdate"`
|
|
RetryAttempts int `json:"retryAttempts"`
|
|
|
|
Complete bool `json:"complete"`
|
|
Failed bool `json:"failed"`
|
|
Status string `json:"status"`
|
|
LastError string `json:"lastError,omitempty"`
|
|
|
|
// Specific job type data:
|
|
Replicate *ReplicateInfo `json:"replicate,omitempty"`
|
|
KeyRotate *KeyRotationInfo `json:"rotation,omitempty"`
|
|
Expired *ExpirationInfo `json:"expired,omitempty"`
|
|
Catalog *CatalogInfo `json:"catalog,omitempty"`
|
|
Untier *UntierInfo `json:"untier,omitempty"`
|
|
}
|
|
|
|
type ReplicateInfo struct {
|
|
// Last bucket/object batch replicated
|
|
Bucket string `json:"lastBucket"`
|
|
Object string `json:"lastObject"`
|
|
|
|
// Verbose information
|
|
Objects int64 `json:"objects"`
|
|
ObjectsFailed int64 `json:"objectsFailed"`
|
|
DeleteMarkers int64 `json:"deleteMarkers"`
|
|
DeleteMarkersFailed int64 `json:"deleteMarkersFailed"`
|
|
BytesTransferred int64 `json:"bytesTransferred"`
|
|
BytesFailed int64 `json:"bytesFailed"`
|
|
}
|
|
|
|
type ExpirationInfo struct {
|
|
// Last bucket/object key rotated
|
|
Bucket string `json:"lastBucket"`
|
|
Object string `json:"lastObject"`
|
|
|
|
// Verbose information
|
|
Objects int64 `json:"objects"`
|
|
ObjectsFailed int64 `json:"objectsFailed"`
|
|
DeleteMarkers int64 `json:"deleteMarkers"`
|
|
DeleteMarkersFailed int64 `json:"deleteMarkersFailed"`
|
|
}
|
|
|
|
type KeyRotationInfo struct {
|
|
// Last bucket/object key rotated
|
|
Bucket string `json:"lastBucket"`
|
|
Object string `json:"lastObject"`
|
|
|
|
// Verbose information
|
|
Objects int64 `json:"objects"`
|
|
ObjectsFailed int64 `json:"objectsFailed"`
|
|
}
|
|
|
|
type CatalogInfo struct {
|
|
Bucket string `json:"bucket"`
|
|
LastBucketScanned string `json:"lastBucketScanned,omitempty"` // Deprecated 07/01/2025; Replaced by `bucket`
|
|
LastObjectScanned string `json:"lastObjectScanned"`
|
|
LastBucketMatched string `json:"lastBucketMatched,omitempty"` // Deprecated 07/01/2025; Replaced by `bucket`
|
|
LastObjectMatched string `json:"lastObjectMatched"`
|
|
|
|
ObjectsScannedCount uint64 `json:"objectsScannedCount"`
|
|
ObjectsMatchedCount uint64 `json:"objectsMatchedCount"`
|
|
|
|
// Represents the number of objects' metadata that were written to output
|
|
// objects.
|
|
RecordsWrittenCount uint64 `json:"recordsWrittenCount"`
|
|
// Represents the number of output objects created.
|
|
OutputObjectsCount uint64 `json:"outputObjectsCount"`
|
|
// Manifest file path (part of the output of a catalog job)
|
|
ManifestPathBucket string `json:"manifestPathBucket"`
|
|
ManifestPathObject string `json:"manifestPathObject"`
|
|
|
|
// Error message
|
|
ErrorMsg string `json:"errorMsg"`
|
|
|
|
// Used to resume catalog jobs
|
|
LastObjectWritten string `json:"lastObjectWritten,omitempty"`
|
|
OutputFiles []CatalogDataFile `json:"outputFiles,omitempty"`
|
|
}
|
|
|
|
// UntierInfo contains progress metrics for a batch untier job.
|
|
type UntierInfo struct {
|
|
Bucket string `json:"bucket"`
|
|
LastObject string `json:"lastObject"`
|
|
Objects int64 `json:"objects"`
|
|
ObjectsFailed int64 `json:"objectsFailed"`
|
|
BytesTransferred int64 `json:"bytesTransferred"`
|
|
BytesFailed int64 `json:"bytesFailed"`
|
|
}
|
|
|
|
// Merge other into 'o'.
|
|
func (o *BatchJobMetrics) Merge(other *BatchJobMetrics) {
|
|
if other == nil || len(other.Jobs) == 0 {
|
|
return
|
|
}
|
|
if o.CollectedAt.Before(other.CollectedAt) {
|
|
// Use latest timestamp
|
|
o.CollectedAt = other.CollectedAt
|
|
}
|
|
// Use latest metrics
|
|
if o.Jobs == nil {
|
|
o.Jobs = make(map[string]JobMetric, len(other.Jobs))
|
|
}
|
|
for k, v := range other.Jobs {
|
|
if exists, ok := o.Jobs[k]; !ok || exists.LastUpdate.Before(v.LastUpdate) {
|
|
o.Jobs[k] = v
|
|
}
|
|
}
|
|
}
|
|
|
|
// SiteResyncMetrics contains metrics for site resync operation
|
|
type SiteResyncMetrics struct {
|
|
// Time these metrics were collected
|
|
CollectedAt time.Time `json:"collected"`
|
|
// Status of resync operation
|
|
ResyncStatus string `json:"resyncStatus,omitempty"`
|
|
StartTime time.Time `json:"startTime"`
|
|
LastUpdate time.Time `json:"lastUpdate"`
|
|
NumBuckets int64 `json:"numBuckets"`
|
|
ResyncID string `json:"resyncID"`
|
|
DeplID string `json:"deplID"`
|
|
|
|
// Completed size in bytes
|
|
ReplicatedSize int64 `json:"completedReplicationSize"`
|
|
// Total number of objects replicated
|
|
ReplicatedCount int64 `json:"replicationCount"`
|
|
// Failed size in bytes
|
|
FailedSize int64 `json:"failedReplicationSize"`
|
|
// Total number of failed operations
|
|
FailedCount int64 `json:"failedReplicationCount"`
|
|
// Buckets that could not be synced
|
|
FailedBuckets []string `json:"failedBuckets"`
|
|
// Last bucket/object replicated.
|
|
Bucket string `json:"bucket,omitempty"`
|
|
Object string `json:"object,omitempty"`
|
|
}
|
|
|
|
func (o SiteResyncMetrics) Complete() bool {
|
|
return strings.ToLower(o.ResyncStatus) == "completed"
|
|
}
|
|
|
|
// Merge other into 'o'.
|
|
func (o *SiteResyncMetrics) Merge(other *SiteResyncMetrics) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
if o.CollectedAt.Before(other.CollectedAt) {
|
|
// Use latest
|
|
*o = *other
|
|
}
|
|
}
|
|
|
|
// SegmentedInterfaceStats is Time segmented interface stats.
|
|
type SegmentedInterfaceStats = Segmented[InterfaceStats, *InterfaceStats]
|
|
|
|
type NetMetrics struct {
|
|
// Time these metrics were collected
|
|
CollectedAt time.Time `json:"collected"`
|
|
|
|
// NICs contains interface -> stats map.
|
|
Interfaces map[string]InterfaceStats
|
|
|
|
// Last day delta statistics.
|
|
LastDay *SegmentedInterfaceStats `json:"last_day,omitempty"`
|
|
|
|
// Last hour delta statistics (1-min segments).
|
|
LastHour *SegmentedInterfaceStats `json:"last_hour,omitempty"`
|
|
|
|
// Deprecated: Does not merge.
|
|
InterfaceName string `json:"interfaceName"`
|
|
|
|
// Internode Stats.
|
|
NetStats procfs.NetDevLine `json:"netstats"`
|
|
}
|
|
|
|
//msgp:replace procfs.NetDevLine with:procfsNetDevLine
|
|
|
|
// Merge other into 'o'.
|
|
func (n *NetMetrics) Merge(other *NetMetrics) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
if n.CollectedAt.Before(other.CollectedAt) {
|
|
// Use latest timestamp
|
|
n.CollectedAt = other.CollectedAt
|
|
}
|
|
for k, v := range other.Interfaces {
|
|
if n.Interfaces == nil {
|
|
n.Interfaces = make(map[string]InterfaceStats, len(other.Interfaces))
|
|
}
|
|
n.Interfaces[k] = n.Interfaces[k].add(v)
|
|
}
|
|
if other.LastDay != nil && n.LastDay == nil {
|
|
n.LastDay = new(SegmentedInterfaceStats)
|
|
}
|
|
n.LastDay.Add(other.LastDay)
|
|
if other.LastHour != nil && n.LastHour == nil {
|
|
n.LastHour = new(SegmentedInterfaceStats)
|
|
}
|
|
n.LastHour.Add(other.LastHour)
|
|
n.NetStats = procfs.NetDevLine(procfsNetDevLine(n.NetStats).add(procfsNetDevLine(other.NetStats)))
|
|
}
|
|
|
|
// InterfaceStats contains accumulated stats for a network interface.
|
|
type InterfaceStats struct {
|
|
N int `json:"n"`
|
|
procfs.NetDevLine `json:"stats"`
|
|
}
|
|
|
|
func (n *InterfaceStats) Add(other *InterfaceStats) {
|
|
if other == nil || n == nil || other.N == 0 {
|
|
return
|
|
}
|
|
n.N = n.N + other.N
|
|
n.NetDevLine = procfs.NetDevLine(procfsNetDevLine(n.NetDevLine).add(procfsNetDevLine(other.NetDevLine)))
|
|
}
|
|
|
|
func (n InterfaceStats) add(other InterfaceStats) InterfaceStats {
|
|
return InterfaceStats{
|
|
N: n.N + other.N,
|
|
NetDevLine: procfs.NetDevLine(procfsNetDevLine(n.NetDevLine).add(procfsNetDevLine(other.NetDevLine))),
|
|
}
|
|
}
|
|
|
|
//msgp:replace NodeCommon with:nodeCommon
|
|
|
|
// nodeCommon - use as replacement for NodeCommon
|
|
// We do not want to give NodeCommon codegen, since it is used for embedding.
|
|
type nodeCommon struct {
|
|
Addr string `json:"addr"`
|
|
Error string `json:"error,omitempty"`
|
|
}
|
|
|
|
type MemMetrics struct {
|
|
// Time these metrics were collected
|
|
CollectedAt time.Time `json:"collected"`
|
|
|
|
Nodes int `json:"nodes"` // Note: Will be zero for older servers.
|
|
|
|
Info MemInfo `json:"memInfo"`
|
|
|
|
LastDay *SegmentedMemMetrics `json:"lastDay,omitempty"`
|
|
|
|
// Last hour statistics (1-min segments).
|
|
LastHour *SegmentedMemMetrics `json:"lastHour,omitempty"`
|
|
}
|
|
|
|
// Merge other into 'm'.
|
|
func (m *MemMetrics) Merge(other *MemMetrics) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
m.Nodes += other.Nodes
|
|
if m.CollectedAt.Before(other.CollectedAt) {
|
|
// Use latest timestamp
|
|
m.CollectedAt = other.CollectedAt
|
|
}
|
|
m.Info.Merge(&other.Info)
|
|
if other.LastDay != nil {
|
|
if m.LastDay == nil {
|
|
m.LastDay = new(SegmentedMemMetrics)
|
|
}
|
|
m.LastDay.Add(other.LastDay)
|
|
}
|
|
if other.LastHour != nil {
|
|
if m.LastHour == nil {
|
|
m.LastHour = new(SegmentedMemMetrics)
|
|
}
|
|
m.LastHour.Add(other.LastHour)
|
|
}
|
|
}
|
|
|
|
// MemInfo contains system's RAM and swap information.
|
|
type MemInfo struct {
|
|
// NodeCommon shouldn't be used since it cannot be merged.
|
|
NodeCommon
|
|
|
|
Total uint64 `json:"total,omitempty"`
|
|
Used uint64 `json:"used,omitempty"`
|
|
Free uint64 `json:"free,omitempty"`
|
|
Available uint64 `json:"available,omitempty"`
|
|
Shared uint64 `json:"shared,omitempty"`
|
|
Cache uint64 `json:"cache,omitempty"`
|
|
Buffers uint64 `json:"buffer,omitempty"`
|
|
SwapSpaceTotal uint64 `json:"swap_space_total,omitempty"`
|
|
SwapSpaceFree uint64 `json:"swap_space_free,omitempty"`
|
|
// Limit will store cgroup limit if configured and
|
|
// less than Total, otherwise same as Total
|
|
Limit uint64 `json:"limit,omitempty"`
|
|
}
|
|
|
|
func (m *MemInfo) Merge(other *MemInfo) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
if m.Total == 0 && m.Addr == "" {
|
|
m.NodeCommon = other.NodeCommon
|
|
} else if m.NodeCommon != other.NodeCommon {
|
|
m.NodeCommon = NodeCommon{}
|
|
}
|
|
m.Total += other.Total
|
|
m.Used += other.Used
|
|
m.Free += other.Free
|
|
m.Available += other.Available
|
|
m.Shared += other.Shared
|
|
m.Cache += other.Cache
|
|
m.Buffers += other.Buffers
|
|
m.SwapSpaceTotal += other.SwapSpaceTotal
|
|
m.SwapSpaceFree += other.SwapSpaceFree
|
|
m.Limit += other.Limit
|
|
}
|
|
|
|
// MemSegment contains compact memory metrics for time-series segmentation.
|
|
type MemSegment struct {
|
|
Used uint64 `json:"used,omitempty"`
|
|
Free uint64 `json:"free,omitempty"`
|
|
Available uint64 `json:"available,omitempty"`
|
|
Limit uint64 `json:"limit,omitempty"`
|
|
N int `json:"n"`
|
|
}
|
|
|
|
// Add other to m for Segmenter interface.
|
|
func (m *MemSegment) Add(other *MemSegment) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
m.Used += other.Used
|
|
m.Free += other.Free
|
|
m.Available += other.Available
|
|
m.Limit += other.Limit
|
|
m.N += other.N
|
|
}
|
|
|
|
// SegmentedMemMetrics are time-segmented memory metrics.
|
|
type SegmentedMemMetrics = Segmented[MemSegment, *MemSegment]
|
|
|
|
// CPUSegment stores CPU time breakdown for a single time segment.
|
|
type CPUSegment struct {
|
|
User float64 `json:"user,omitempty"`
|
|
System float64 `json:"system,omitempty"`
|
|
Idle float64 `json:"idle,omitempty"`
|
|
Nice float64 `json:"nice,omitempty"`
|
|
Iowait float64 `json:"iowait,omitempty"`
|
|
Irq float64 `json:"irq,omitempty"`
|
|
Softirq float64 `json:"softirq,omitempty"`
|
|
Steal float64 `json:"steal,omitempty"`
|
|
Guest float64 `json:"guest,omitempty"`
|
|
GuestNice float64 `json:"guestNice,omitempty"`
|
|
N int `json:"n"`
|
|
}
|
|
|
|
// Add other to c for Segmenter interface.
|
|
func (c *CPUSegment) Add(other *CPUSegment) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
c.User += other.User
|
|
c.System += other.System
|
|
c.Idle += other.Idle
|
|
c.Nice += other.Nice
|
|
c.Iowait += other.Iowait
|
|
c.Irq += other.Irq
|
|
c.Softirq += other.Softirq
|
|
c.Steal += other.Steal
|
|
c.Guest += other.Guest
|
|
c.GuestNice += other.GuestNice
|
|
c.N += other.N
|
|
}
|
|
|
|
// SegmentedCPUMetrics are time-segmented CPU metrics.
|
|
type SegmentedCPUMetrics = Segmented[CPUSegment, *CPUSegment]
|
|
|
|
// PowerSegment stores power draw for a single time segment.
|
|
// Each node normalizes to a single sample (N=1) before reporting,
|
|
// so N equals the number of contributing nodes after merge.
|
|
type PowerSegment struct {
|
|
SumWatts float64 `json:"sumWatts,omitempty"`
|
|
MinWatts float64 `json:"minWatts,omitempty"`
|
|
MaxWatts float64 `json:"maxWatts,omitempty"`
|
|
N int `json:"n"`
|
|
}
|
|
|
|
// Add other to p for Segmenter interface.
|
|
func (p *PowerSegment) Add(other *PowerSegment) {
|
|
if other == nil || other.N == 0 {
|
|
return
|
|
}
|
|
p.SumWatts += other.SumWatts
|
|
if p.N == 0 || other.MinWatts < p.MinWatts {
|
|
p.MinWatts = other.MinWatts
|
|
}
|
|
p.MaxWatts = max(p.MaxWatts, other.MaxWatts)
|
|
p.N += other.N
|
|
}
|
|
|
|
// SegmentedPowerMetrics are time-segmented power draw metrics.
|
|
type SegmentedPowerMetrics = Segmented[PowerSegment, *PowerSegment]
|
|
|
|
//msgp:replace cpu.TimesStat with:cpuTimesStat
|
|
//msgp:replace load.AvgStat with:loadAvgStat
|
|
|
|
type CPUMetrics struct {
|
|
// Time these metrics were collected
|
|
CollectedAt time.Time `json:"collected"`
|
|
|
|
Nodes int `json:"nodes"` // Note: May be unset for older servers.
|
|
|
|
TimesStat cpu.TimesStat `json:"timesStat2"`
|
|
TimesCount int `json:"timesCount,omitempty"`
|
|
LoadStat load.AvgStat `json:"loadStat2"`
|
|
LoadStatCount int `json:"loadCount,omitempty"`
|
|
CPUCount int `json:"cpuCount,omitempty"`
|
|
|
|
LastDay *SegmentedCPUMetrics `json:"lastDay,omitempty"`
|
|
|
|
// Last hour statistics (1-min segments).
|
|
LastHour *SegmentedCPUMetrics `json:"lastHour,omitempty"`
|
|
|
|
// Aggregated CPU information
|
|
CPUByModel map[string]int `json:"cpu_by_model,omitempty"` // ModelName -> count of CPUs
|
|
TotalMhz float64 `json:"total_mhz,omitempty"` // Accumulated MHz
|
|
TotalCores int `json:"total_cores,omitempty"` // Accumulated cores
|
|
TotalCacheSize int64 `json:"total_cache_size,omitempty"` // Accumulated cache size in bytes
|
|
|
|
// Aggregated CPU frequency information
|
|
FreqStatsCount int `json:"freq_stats_count,omitempty"` // Number of freq stats (for averaging)
|
|
GovernorFreq map[string]int `json:"governor_freq,omitempty"` // Governor -> count
|
|
TotalCurrentFreq uint64 `json:"total_current_freq,omitempty"` // Accumulated current freq
|
|
TotalScalingCurrentFreq uint64 `json:"total_scaling_current_freq,omitempty"` // Accumulated scaling current freq
|
|
MinCPUInfoFreq uint64 `json:"min_freq,omitempty"` // Minimum of CpuinfoMinimumFrequency
|
|
MaxCPUInfoFreq uint64 `json:"max_freq,omitempty"` // Maximum of CpuinfoMaximumFrequency
|
|
MinScalingFreq uint64 `json:"min_scaling_freq,omitempty"` // Minimum of ScalingMinimumFrequency
|
|
MaxScalingFreq uint64 `json:"max_scaling_freq,omitempty"` // Maximum of ScalingMaximumFrequency
|
|
|
|
// Power draw metrics (from IPMI/BMC, omitted when unavailable)
|
|
PowerNodes int `json:"power_nodes,omitempty"`
|
|
TotalWatts float64 `json:"total_watts,omitempty"`
|
|
MinNodeWatts float64 `json:"min_node_watts,omitempty"`
|
|
MaxNodeWatts float64 `json:"max_node_watts,omitempty"`
|
|
PowerSourceCounts map[string]int `json:"power_source_counts,omitempty"`
|
|
PowerLastDay *SegmentedPowerMetrics `json:"powerLastDay,omitempty"`
|
|
PowerLastHour *SegmentedPowerMetrics `json:"powerLastHour,omitempty"`
|
|
}
|
|
|
|
// Merge other into 'm'.
|
|
func (m *CPUMetrics) Merge(other *CPUMetrics) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
m.Nodes += other.Nodes
|
|
if m.CollectedAt.Before(other.CollectedAt) {
|
|
// Use latest timestamp
|
|
m.CollectedAt = other.CollectedAt
|
|
}
|
|
|
|
m.TimesStat.User += other.TimesStat.User
|
|
m.TimesStat.System += other.TimesStat.System
|
|
m.TimesStat.Idle += other.TimesStat.Idle
|
|
m.TimesStat.Nice += other.TimesStat.Nice
|
|
m.TimesStat.Iowait += other.TimesStat.Iowait
|
|
m.TimesStat.Irq += other.TimesStat.Irq
|
|
m.TimesStat.Softirq += other.TimesStat.Softirq
|
|
m.TimesStat.Steal += other.TimesStat.Steal
|
|
m.TimesStat.Guest += other.TimesStat.Guest
|
|
m.TimesStat.GuestNice += other.TimesStat.GuestNice
|
|
m.TimesCount += other.TimesCount
|
|
m.LoadStat.Load1 += other.LoadStat.Load1
|
|
m.LoadStat.Load5 += other.LoadStat.Load5
|
|
m.LoadStat.Load15 += other.LoadStat.Load15
|
|
m.LoadStatCount += other.LoadStatCount
|
|
m.CPUCount += other.CPUCount
|
|
|
|
// Merge aggregated CPU information
|
|
if len(other.CPUByModel) > 0 {
|
|
if m.CPUByModel == nil {
|
|
m.CPUByModel = make(map[string]int)
|
|
}
|
|
for model, count := range other.CPUByModel {
|
|
m.CPUByModel[model] += count
|
|
}
|
|
}
|
|
m.TotalMhz += other.TotalMhz
|
|
m.TotalCores += other.TotalCores
|
|
m.TotalCacheSize += other.TotalCacheSize
|
|
|
|
// Merge aggregated CPU frequency information
|
|
if len(other.GovernorFreq) > 0 {
|
|
if m.GovernorFreq == nil {
|
|
m.GovernorFreq = make(map[string]int)
|
|
}
|
|
for governor, count := range other.GovernorFreq {
|
|
m.GovernorFreq[governor] += count
|
|
}
|
|
}
|
|
m.TotalCurrentFreq += other.TotalCurrentFreq
|
|
m.TotalScalingCurrentFreq += other.TotalScalingCurrentFreq
|
|
|
|
// Handle min/max frequencies properly
|
|
// Use FreqStatsCount to determine if this is the first merge
|
|
if other.MinCPUInfoFreq > 0 {
|
|
if m.FreqStatsCount == 0 || other.MinCPUInfoFreq < m.MinCPUInfoFreq {
|
|
m.MinCPUInfoFreq = other.MinCPUInfoFreq
|
|
}
|
|
}
|
|
m.MaxCPUInfoFreq = max(other.MaxCPUInfoFreq, m.MaxCPUInfoFreq)
|
|
if other.MinScalingFreq > 0 {
|
|
if m.FreqStatsCount == 0 || other.MinScalingFreq < m.MinScalingFreq {
|
|
m.MinScalingFreq = other.MinScalingFreq
|
|
}
|
|
}
|
|
if other.MaxScalingFreq > m.MaxScalingFreq {
|
|
m.MaxScalingFreq = other.MaxScalingFreq
|
|
}
|
|
|
|
m.FreqStatsCount += other.FreqStatsCount
|
|
|
|
if other.LastDay != nil {
|
|
if m.LastDay == nil {
|
|
m.LastDay = new(SegmentedCPUMetrics)
|
|
}
|
|
m.LastDay.Add(other.LastDay)
|
|
}
|
|
if other.LastHour != nil {
|
|
if m.LastHour == nil {
|
|
m.LastHour = new(SegmentedCPUMetrics)
|
|
}
|
|
m.LastHour.Add(other.LastHour)
|
|
}
|
|
|
|
// Merge power draw metrics
|
|
if other.PowerNodes > 0 {
|
|
if m.PowerNodes == 0 || other.MinNodeWatts < m.MinNodeWatts {
|
|
m.MinNodeWatts = other.MinNodeWatts
|
|
}
|
|
m.MaxNodeWatts = max(m.MaxNodeWatts, other.MaxNodeWatts)
|
|
m.TotalWatts += other.TotalWatts
|
|
m.PowerNodes += other.PowerNodes
|
|
if len(other.PowerSourceCounts) > 0 {
|
|
if m.PowerSourceCounts == nil {
|
|
m.PowerSourceCounts = make(map[string]int)
|
|
}
|
|
for src, count := range other.PowerSourceCounts {
|
|
m.PowerSourceCounts[src] += count
|
|
}
|
|
}
|
|
}
|
|
if other.PowerLastDay != nil {
|
|
if m.PowerLastDay == nil {
|
|
m.PowerLastDay = new(SegmentedPowerMetrics)
|
|
}
|
|
m.PowerLastDay.Add(other.PowerLastDay)
|
|
}
|
|
if other.PowerLastHour != nil {
|
|
if m.PowerLastHour == nil {
|
|
m.PowerLastHour = new(SegmentedPowerMetrics)
|
|
}
|
|
m.PowerLastHour.Add(other.PowerLastHour)
|
|
}
|
|
}
|
|
|
|
// RPCMetrics contains metrics for RPC operations.
|
|
// Metrics are collected on the sender side of RPC calls.
|
|
type RPCMetrics struct {
|
|
Nodes int `json:"nodes,omitempty"`
|
|
|
|
CollectedAt time.Time `json:"collected"`
|
|
|
|
// Connection stats accumulated for grid systems running on nodes.
|
|
//nolint:staticcheck // SA5008
|
|
ConnectionStats `json:",flatten"`
|
|
|
|
// Last minute operation statistics by handler.
|
|
LastMinute map[string]RPCStats `json:"lastMinute,omitempty"`
|
|
|
|
// Last day operation statistics by handler, segmented.
|
|
LastDay map[string]SegmentedRPCMetrics `json:"lastDay,omitempty"`
|
|
|
|
ByDestination map[string]ConnectionStats `json:"byDestination,omitempty"`
|
|
ByCaller map[string]ConnectionStats `json:"byCaller,omitempty"`
|
|
}
|
|
|
|
// Merge other into 'm'.
|
|
func (m *RPCMetrics) Merge(other *RPCMetrics) {
|
|
if m == nil || other == nil {
|
|
return
|
|
}
|
|
m.Nodes += other.Nodes
|
|
if m.CollectedAt.Before(other.CollectedAt) {
|
|
// Use latest timestamp
|
|
m.CollectedAt = other.CollectedAt
|
|
}
|
|
|
|
m.ConnectionStats.Merge(&other.ConnectionStats)
|
|
|
|
for k, v := range other.ByDestination {
|
|
if m.ByDestination == nil {
|
|
m.ByDestination = make(map[string]ConnectionStats, len(other.ByDestination))
|
|
}
|
|
existing := m.ByDestination[k]
|
|
existing.Merge(&v)
|
|
m.ByDestination[k] = existing
|
|
}
|
|
|
|
for k, v := range other.ByCaller {
|
|
if m.ByCaller == nil {
|
|
m.ByCaller = make(map[string]ConnectionStats, len(other.ByCaller))
|
|
}
|
|
existing := m.ByCaller[k]
|
|
existing.Merge(&v)
|
|
m.ByCaller[k] = existing
|
|
}
|
|
|
|
for k, v := range other.LastMinute {
|
|
if m.LastMinute == nil {
|
|
m.LastMinute = make(map[string]RPCStats, len(other.LastMinute))
|
|
}
|
|
existing := m.LastMinute[k]
|
|
existing.Merge(v)
|
|
m.LastMinute[k] = existing
|
|
}
|
|
for k, v := range other.LastDay {
|
|
if m.LastDay == nil {
|
|
m.LastDay = make(map[string]SegmentedRPCMetrics, len(other.LastDay))
|
|
}
|
|
existing, ok := m.LastDay[k]
|
|
if !ok {
|
|
// Deep copy to avoid sharing slice references
|
|
vCopy := v
|
|
if len(v.Segments) > 0 {
|
|
vCopy.Segments = append([]RPCStats{}, v.Segments...)
|
|
}
|
|
m.LastDay[k] = vCopy
|
|
continue
|
|
}
|
|
existing.Add(&v)
|
|
m.LastDay[k] = existing
|
|
}
|
|
}
|
|
|
|
// LastMinuteTotal returns the total RPCStats for the last minute.
|
|
func (m *RPCMetrics) LastMinuteTotal() RPCStats {
|
|
var res RPCStats
|
|
|
|
// First, check if we have mixed timestamp states across handlers
|
|
hasTimestamps := false
|
|
hasNilTimestamps := false
|
|
for _, stats := range m.LastMinute {
|
|
if stats.StartTime != nil || stats.EndTime != nil {
|
|
hasTimestamps = true
|
|
} else {
|
|
hasNilTimestamps = true
|
|
}
|
|
}
|
|
|
|
// If we have mixed timestamp states, we need to nullify them during merge
|
|
if hasTimestamps && hasNilTimestamps {
|
|
for _, stats := range m.LastMinute {
|
|
// Create a copy without timestamps to merge
|
|
cleanStats := stats
|
|
cleanStats.StartTime = nil
|
|
cleanStats.EndTime = nil
|
|
res.Merge(cleanStats)
|
|
}
|
|
} else {
|
|
// Normal merge when all handlers have consistent timestamp state
|
|
for _, stats := range m.LastMinute {
|
|
res.Merge(stats)
|
|
}
|
|
}
|
|
|
|
// Since we are merging across APIs must reset track node count.
|
|
return res
|
|
}
|
|
|
|
// LastDayTotalSegmented returns the total SegmentedRPCMetrics for the last day.
|
|
func (m *RPCMetrics) LastDayTotalSegmented() SegmentedRPCMetrics {
|
|
var res SegmentedRPCMetrics
|
|
for _, stats := range m.LastDay {
|
|
res.Add(&stats)
|
|
}
|
|
return res
|
|
}
|
|
|
|
// LastDayTotal returns the accumulated RPCStats for the last day.
|
|
func (m *RPCMetrics) LastDayTotal() RPCStats {
|
|
var res RPCStats
|
|
for _, stats := range m.LastDay {
|
|
for _, s := range stats.Segments {
|
|
res.Merge(s)
|
|
}
|
|
}
|
|
return res
|
|
}
|
|
|
|
// ConnectionStats are the overall connection stats.
|
|
type ConnectionStats struct {
|
|
Connected int `json:"connected,omitempty"`
|
|
Disconnected int `json:"disconnected,omitempty"`
|
|
ReconnectCount int `json:"reconnectCount,omitempty"` // Total reconnects.
|
|
OutgoingStreams int `json:"outgoingStreams,omitempty"`
|
|
IncomingStreams int `json:"incomingStreams,omitempty"`
|
|
OutgoingMessages int64 `json:"outgoingMessages,omitempty"`
|
|
IncomingMessages int64 `json:"incomingMessages,omitempty"`
|
|
OutgoingBytes int64 `json:"outgoingBytes,omitempty"` // Total number of bytes sent.
|
|
IncomingBytes int64 `json:"incomingBytes,omitempty"` // Total number of bytes received.
|
|
OutQueue int `json:"outQueue,omitempty"`
|
|
LastPongTime time.Time `json:"lastPongTime,omitempty"`
|
|
LastConnectTime time.Time `json:"lastConnectTime,omitempty"`
|
|
LastPingMS float64 `json:"lastPingMS,omitempty"`
|
|
MaxPingDurMS float64 `json:"maxPingDurMS,omitempty"` // Maximum across all merged entries.
|
|
}
|
|
|
|
// Merge other into c.
|
|
func (c *ConnectionStats) Merge(other *ConnectionStats) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
c.Connected += other.Connected
|
|
c.Disconnected += other.Disconnected
|
|
c.ReconnectCount += other.ReconnectCount
|
|
c.OutgoingStreams += other.OutgoingStreams
|
|
c.IncomingStreams += other.IncomingStreams
|
|
c.OutgoingMessages += other.OutgoingMessages
|
|
c.IncomingMessages += other.IncomingMessages
|
|
c.OutgoingBytes += other.OutgoingBytes
|
|
c.IncomingBytes += other.IncomingBytes
|
|
c.OutQueue += other.OutQueue
|
|
if c.LastPongTime.Before(other.LastPongTime) {
|
|
c.LastPongTime = other.LastPongTime
|
|
c.LastPingMS = other.LastPingMS
|
|
}
|
|
if c.LastConnectTime.Before(other.LastConnectTime) {
|
|
c.LastConnectTime = other.LastConnectTime
|
|
}
|
|
if c.MaxPingDurMS < other.MaxPingDurMS {
|
|
c.MaxPingDurMS = other.MaxPingDurMS
|
|
}
|
|
}
|
|
|
|
// SegmentedRPCMetrics are segmented RPC metrics.
|
|
type SegmentedRPCMetrics = Segmented[RPCStats, *RPCStats]
|
|
|
|
// RPCStats contains RPC statistics for RPC requests through grid.
|
|
type RPCStats struct {
|
|
StartTime *time.Time `json:"startTime,omitempty"` // Time range this data covers unless merged from sources with different start times..
|
|
EndTime *time.Time `json:"endTime,omitempty"` // Time range this data covers unless merged from sources with different end times.
|
|
WallTimeSecs float64 `json:"wallTimeSecs,omitempty"` // Wall time this data covers, accumulated from all nodes.
|
|
Requests int64 `json:"requests,omitempty"` // Total number of requests.
|
|
RequestTimeSecs float64 `json:"requestTimeSecs,omitempty"` // Total request time.
|
|
IncomingBytes int64 `json:"incomingBytes,omitempty"` // Total number of bytes received.
|
|
OutgoingBytes int64 `json:"outgoingBytes,omitempty"` // Total number of bytes sent.
|
|
}
|
|
|
|
// Add 'other' to a.
|
|
func (a *RPCStats) Add(other *RPCStats) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
a.Merge(*other)
|
|
}
|
|
|
|
// Merge other into 'a'.
|
|
func (a *RPCStats) Merge(other RPCStats) {
|
|
if a.StartTime == nil && a.Requests == 0 {
|
|
a.StartTime = other.StartTime
|
|
}
|
|
if a.EndTime == nil && a.Requests == 0 {
|
|
a.EndTime = other.EndTime
|
|
}
|
|
if a.StartTime != nil && other.StartTime != nil && !a.StartTime.Equal(*other.StartTime) {
|
|
a.StartTime = nil
|
|
}
|
|
if a.EndTime != nil && other.EndTime != nil && !a.EndTime.Equal(*other.EndTime) {
|
|
a.EndTime = nil
|
|
}
|
|
a.WallTimeSecs += other.WallTimeSecs
|
|
a.Requests += other.Requests
|
|
a.IncomingBytes += other.IncomingBytes
|
|
a.OutgoingBytes += other.OutgoingBytes
|
|
a.RequestTimeSecs += other.RequestTimeSecs
|
|
}
|
|
|
|
//msgp:replace metrics.Float64Histogram with:localF64H
|
|
|
|
// local copy of localF64H, can be casted to/from metrics.Float64Histogram
|
|
type localF64H struct {
|
|
Counts []uint64 `json:"counts,omitempty"`
|
|
Buckets []float64 `json:"buckets,omitempty"`
|
|
}
|
|
|
|
// RuntimeMetrics contains metrics for the go runtime.
|
|
// See more at https://pkg.go.dev/runtime/metrics
|
|
type RuntimeMetrics struct {
|
|
// UintMetrics contains KindUint64 values
|
|
UintMetrics map[string]uint64 `json:"uintMetrics,omitempty"`
|
|
|
|
// FloatMetrics contains KindFloat64 values
|
|
FloatMetrics map[string]float64 `json:"floatMetrics,omitempty"`
|
|
|
|
// HistMetrics contains KindFloat64Histogram values
|
|
HistMetrics map[string]metrics.Float64Histogram `json:"histMetrics,omitempty"`
|
|
|
|
// N tracks the number of merged entries.
|
|
N int `json:"n"`
|
|
|
|
LastDay *SegmentedRuntimeMetrics `json:"lastDay,omitempty"`
|
|
|
|
// Last hour statistics (1-min segments).
|
|
LastHour *SegmentedRuntimeMetrics `json:"lastHour,omitempty"`
|
|
}
|
|
|
|
// Merge other into 'm'.
|
|
func (m *RuntimeMetrics) Merge(other *RuntimeMetrics) {
|
|
if m == nil || other == nil {
|
|
return
|
|
}
|
|
if m.UintMetrics == nil {
|
|
m.UintMetrics = make(map[string]uint64, len(other.UintMetrics))
|
|
}
|
|
if m.FloatMetrics == nil {
|
|
m.FloatMetrics = make(map[string]float64, len(other.FloatMetrics))
|
|
}
|
|
if m.HistMetrics == nil {
|
|
m.HistMetrics = make(map[string]metrics.Float64Histogram, len(other.HistMetrics))
|
|
}
|
|
for k, v := range other.UintMetrics {
|
|
m.UintMetrics[k] += v
|
|
}
|
|
for k, v := range other.FloatMetrics {
|
|
m.FloatMetrics[k] += v
|
|
}
|
|
for k, v := range other.HistMetrics {
|
|
existing := m.HistMetrics[k]
|
|
if len(existing.Buckets) == 0 {
|
|
m.HistMetrics[k] = v
|
|
continue
|
|
}
|
|
// TODO: Technically, I guess we may have differing buckets,
|
|
// but they should be the same for the runtime.
|
|
if len(existing.Buckets) == len(v.Buckets) {
|
|
for i, count := range v.Counts {
|
|
existing.Counts[i] += count
|
|
}
|
|
}
|
|
}
|
|
m.N += other.N
|
|
if other.LastDay != nil {
|
|
if m.LastDay == nil {
|
|
m.LastDay = new(SegmentedRuntimeMetrics)
|
|
}
|
|
m.LastDay.Add(other.LastDay)
|
|
}
|
|
if other.LastHour != nil {
|
|
if m.LastHour == nil {
|
|
m.LastHour = new(SegmentedRuntimeMetrics)
|
|
}
|
|
m.LastHour.Add(other.LastHour)
|
|
}
|
|
}
|
|
|
|
// RuntimeSegment contains compact runtime metrics for time-series segmentation.
|
|
type RuntimeSegment struct {
|
|
UintMetrics map[string]uint64 `json:"uintMetrics,omitempty"`
|
|
FloatMetrics map[string]float64 `json:"floatMetrics,omitempty"`
|
|
N int `json:"n"`
|
|
}
|
|
|
|
// Add other to r for Segmenter interface.
|
|
func (r *RuntimeSegment) Add(other *RuntimeSegment) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
if r.UintMetrics == nil && len(other.UintMetrics) > 0 {
|
|
r.UintMetrics = make(map[string]uint64, len(other.UintMetrics))
|
|
}
|
|
if r.FloatMetrics == nil && len(other.FloatMetrics) > 0 {
|
|
r.FloatMetrics = make(map[string]float64, len(other.FloatMetrics))
|
|
}
|
|
for k, v := range other.UintMetrics {
|
|
r.UintMetrics[k] += v
|
|
}
|
|
for k, v := range other.FloatMetrics {
|
|
r.FloatMetrics[k] += v
|
|
}
|
|
r.N += other.N
|
|
}
|
|
|
|
// SegmentedRuntimeMetrics are time-segmented runtime metrics.
|
|
type SegmentedRuntimeMetrics = Segmented[RuntimeSegment, *RuntimeSegment]
|
|
|
|
// APIStats contains accumulated statistics for the API on a number of nodes.
|
|
type APIStats struct {
|
|
Nodes int `json:"nodes,omitempty"` // Number of nodes that have reported data.
|
|
StartTime *time.Time `json:"startTime,omitempty"` // Time range this data covers unless merged from sources with different start times..
|
|
EndTime *time.Time `json:"endTime,omitempty"` // Time range this data covers unless merged from sources with different end times.
|
|
WallTimeSecs float64 `json:"wallTimeSecs,omitempty"` // Wall time this data covers, accumulated from all nodes.
|
|
Requests int64 `json:"requests,omitempty"` // Total number of requests.
|
|
IncomingBytes int64 `json:"incomingBytes,omitempty"` // Total number of bytes received.
|
|
OutgoingBytes int64 `json:"outgoingBytes,omitempty"` // Total number of bytes sent.
|
|
Errors4xx int `json:"errors_4xx,omitempty"` // Total number of 4xx (client request) errors.
|
|
Errors5xx int `json:"errors_5xx,omitempty"` // Total number of 5xx (serverside) errors.
|
|
Canceled int64 `json:"canceled,omitempty"` // Requests that were canceled before they finished processing.
|
|
|
|
// Request times
|
|
RequestTimeSecs float64 `json:"requestTimeSecs,omitempty"` // Total request time.
|
|
ReqReadSecs float64 `json:"reqReadSecs,omitempty"` // Total time spent on request reads in seconds.
|
|
RespSecs float64 `json:"respSecs,omitempty"` // Total time spent on responses output writes in seconds.
|
|
RespTTFBSecs float64 `json:"respTtfbSecs,omitempty"` // Total time spent on TTFB (req read -> response first byte) in seconds.
|
|
ReadBlockedSecs float64 `json:"readBlocked,omitempty"` // Time spent waiting for reads from client.
|
|
WriteBlockedSecs float64 `json:"writeBlocked,omitempty"` // Time spent waiting for writes to client.
|
|
|
|
// Request times min/max
|
|
RequestTimeSecsMin float64 `json:"requestTimeSecsMin,omitempty"` // Min request time.
|
|
RequestTimeSecsMax float64 `json:"requestTimeSecsMax,omitempty"` // Max request time.
|
|
ReqReadSecsMin float64 `json:"reqReadSecsMin,omitempty"` // Min time spent on request reads in seconds.
|
|
ReqReadSecsMax float64 `json:"reqReadSecsMax,omitempty"` // Max time spent on request reads in seconds.
|
|
RespSecsMin float64 `json:"respSecsMin,omitempty"` // Min time spent on responses writes in seconds.
|
|
RespSecsMax float64 `json:"respSecsMax,omitempty"` // Max time spent on responses writes in seconds.
|
|
RespTTFBSecsMin float64 `json:"respTtfbSecsMin,omitempty"` // Min time spent on TTFB (req read -> response first byte) in seconds.
|
|
RespTTFBSecsMax float64 `json:"respTtfbSecsMax,omitempty"` // Max time spent on TTFB (req read -> response first byte) in seconds.
|
|
|
|
Rejected RejectedAPIStats `json:"rejected,omitempty"`
|
|
}
|
|
|
|
// RejectedAPIStats contains statistics for rejected requests.
|
|
type RejectedAPIStats struct {
|
|
Auth int64 `json:"auth,omitempty"` // Total number of rejected authentication requests.
|
|
RequestsTime int64 `json:"requestsTime,omitempty"` // Requests that were rejected due to outdated request signature.
|
|
Header int64 `json:"header,omitempty"` // Requests that were rejected due to header signature.
|
|
Invalid int64 `json:"invalid,omitempty"` // Requests that were rejected due to invalid request signature.
|
|
NotImplemented int64 `json:"notImplemented,omitempty"` // Requests that were rejected due to not implemented API.
|
|
}
|
|
|
|
// Add 'other' to a.
|
|
func (a *APIStats) Add(other *APIStats) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
a.Merge(*other)
|
|
}
|
|
|
|
// Merge other into 'a'.
|
|
func (a *APIStats) Merge(other APIStats) {
|
|
if a.StartTime == nil && a.Requests == 0 {
|
|
a.StartTime = other.StartTime
|
|
}
|
|
if a.EndTime == nil && a.Requests == 0 {
|
|
a.EndTime = other.EndTime
|
|
}
|
|
if a.StartTime != nil && other.StartTime != nil && !a.StartTime.Equal(*other.StartTime) {
|
|
a.StartTime = nil
|
|
}
|
|
if a.EndTime != nil && other.EndTime != nil && !a.EndTime.Equal(*other.EndTime) {
|
|
a.EndTime = nil
|
|
}
|
|
|
|
a.Nodes += other.Nodes
|
|
a.WallTimeSecs += other.WallTimeSecs
|
|
a.Requests += other.Requests
|
|
a.IncomingBytes += other.IncomingBytes
|
|
a.OutgoingBytes += other.OutgoingBytes
|
|
a.RequestTimeSecs += other.RequestTimeSecs
|
|
a.ReqReadSecs += other.ReqReadSecs
|
|
a.RespSecs += other.RespSecs
|
|
a.RespTTFBSecs += other.RespTTFBSecs
|
|
a.Errors4xx += other.Errors4xx
|
|
a.Errors5xx += other.Errors5xx
|
|
a.Canceled += other.Canceled
|
|
a.Rejected.Auth += other.Rejected.Auth
|
|
a.Rejected.RequestsTime += other.Rejected.RequestsTime
|
|
a.Rejected.Header += other.Rejected.Header
|
|
a.Rejected.Invalid += other.Rejected.Invalid
|
|
a.Rejected.NotImplemented += other.Rejected.NotImplemented
|
|
a.ReadBlockedSecs += other.ReadBlockedSecs
|
|
a.WriteBlockedSecs += other.WriteBlockedSecs
|
|
|
|
if a.Requests == 0 && other.Requests == 0 {
|
|
return
|
|
}
|
|
|
|
// Find 2 to min/max. If we have 1, just use that twice
|
|
at := *a
|
|
bt := other
|
|
if a.Requests == other.Requests {
|
|
at = bt
|
|
}
|
|
if other.Requests == 0 {
|
|
bt = at
|
|
}
|
|
a.RequestTimeSecsMin = min(at.RequestTimeSecsMin, bt.RequestTimeSecsMin)
|
|
a.RequestTimeSecsMax = max(at.RequestTimeSecsMax, bt.RequestTimeSecsMax)
|
|
a.ReqReadSecsMin = min(at.ReqReadSecsMin, bt.ReqReadSecsMin)
|
|
a.ReqReadSecsMax = max(at.ReqReadSecsMax, bt.ReqReadSecsMax)
|
|
a.RespSecsMin = min(at.RespSecsMin, bt.RespSecsMin)
|
|
a.RespSecsMax = max(at.RespSecsMax, bt.RespSecsMax)
|
|
a.RespTTFBSecsMin = min(at.RespTTFBSecsMin, bt.RespTTFBSecsMin)
|
|
a.RespTTFBSecsMax = max(at.RespTTFBSecsMax, bt.RespTTFBSecsMax)
|
|
}
|
|
|
|
// SegmentedAPIMetrics are segmented API metrics.
|
|
type SegmentedAPIMetrics = Segmented[APIStats, *APIStats]
|
|
|
|
// APIMetrics contains metrics for API operations.
|
|
type APIMetrics struct {
|
|
// Time these metrics were collected
|
|
CollectedAt time.Time `json:"collected"`
|
|
|
|
// Nodes responded to the request.
|
|
Nodes int `json:"nodes"`
|
|
|
|
// Number of active requests.
|
|
ActiveRequests int64 `json:"activeRequests,omitempty"`
|
|
|
|
// Number of queued requests.
|
|
QueuedRequests int64 `json:"queuedRequests,omitempty"`
|
|
|
|
// Last minute operation statistics by API.
|
|
LastMinuteAPI map[string]APIStats `json:"lastMinuteApi,omitempty"`
|
|
|
|
// Last hour operation statistics by API, segmented (1-min intervals).
|
|
LastHourAPI map[string]SegmentedAPIMetrics `json:"lastHourApi,omitempty"`
|
|
|
|
// Last day operation statistics by API, segmented (15-min intervals).
|
|
LastDayAPI map[string]SegmentedAPIMetrics `json:"lastDayApi,omitempty"`
|
|
|
|
// SinceStart contains operation statistics since server(s) started.
|
|
SinceStart APIStats `json:"since_start"`
|
|
}
|
|
|
|
func (a *APIMetrics) Merge(b *APIMetrics) {
|
|
if b == nil {
|
|
return
|
|
}
|
|
if a.CollectedAt.Before(b.CollectedAt) {
|
|
a.CollectedAt = b.CollectedAt
|
|
}
|
|
a.Nodes += b.Nodes
|
|
a.ActiveRequests += b.ActiveRequests
|
|
a.QueuedRequests += b.QueuedRequests
|
|
|
|
for k, v := range b.LastMinuteAPI {
|
|
if a.LastMinuteAPI == nil {
|
|
a.LastMinuteAPI = make(map[string]APIStats, len(b.LastMinuteAPI))
|
|
}
|
|
existing := a.LastMinuteAPI[k]
|
|
existing.Merge(v)
|
|
a.LastMinuteAPI[k] = existing
|
|
}
|
|
for k, v := range b.LastHourAPI {
|
|
if a.LastHourAPI == nil {
|
|
a.LastHourAPI = make(map[string]SegmentedAPIMetrics, len(b.LastHourAPI))
|
|
}
|
|
existing, ok := a.LastHourAPI[k]
|
|
if !ok {
|
|
vCopy := v
|
|
if len(v.Segments) > 0 {
|
|
vCopy.Segments = append([]APIStats{}, v.Segments...)
|
|
}
|
|
a.LastHourAPI[k] = vCopy
|
|
continue
|
|
}
|
|
existing.Add(&v)
|
|
a.LastHourAPI[k] = existing
|
|
}
|
|
for k, v := range b.LastDayAPI {
|
|
if a.LastDayAPI == nil {
|
|
a.LastDayAPI = make(map[string]SegmentedAPIMetrics, len(b.LastDayAPI))
|
|
}
|
|
existing, ok := a.LastDayAPI[k]
|
|
if !ok {
|
|
vCopy := v
|
|
if len(v.Segments) > 0 {
|
|
vCopy.Segments = append([]APIStats{}, v.Segments...)
|
|
}
|
|
a.LastDayAPI[k] = vCopy
|
|
continue
|
|
}
|
|
existing.Add(&v)
|
|
a.LastDayAPI[k] = existing
|
|
}
|
|
a.SinceStart.Merge(b.SinceStart)
|
|
}
|
|
|
|
// LastMinuteTotal returns the total APIStats for the last minute.
|
|
func (a APIMetrics) LastMinuteTotal() APIStats {
|
|
var res APIStats
|
|
for _, stats := range a.LastMinuteAPI {
|
|
res.Merge(stats)
|
|
}
|
|
// Since we are merging across APIs must reset track node count.
|
|
res.Nodes = a.Nodes
|
|
return res
|
|
}
|
|
|
|
// LastDayTotalSegmented returns the total SegmentedAPIMetrics for the last day.
|
|
// There will be no node-count for values.
|
|
func (a APIMetrics) LastDayTotalSegmented() SegmentedAPIMetrics {
|
|
var res SegmentedAPIMetrics
|
|
for _, stats := range a.LastDayAPI {
|
|
res.Add(&stats)
|
|
}
|
|
// Since we are merging across APIs must reset track node count.
|
|
for i := range res.Segments {
|
|
res.Segments[i].Nodes = a.Nodes
|
|
}
|
|
return res
|
|
}
|
|
|
|
// LastDayTotal returns the accumulated APIStats for the last day.
|
|
func (a APIMetrics) LastDayTotal() APIStats {
|
|
var res APIStats
|
|
for _, stats := range a.LastDayAPI {
|
|
for _, s := range stats.Segments {
|
|
res.Merge(s)
|
|
}
|
|
}
|
|
// Since we are merging across APIs must reset track node count.
|
|
res.Nodes = a.Nodes
|
|
|
|
return res
|
|
}
|
|
|
|
// Segmenter implement interface on pointers.
|
|
type Segmenter[T any] interface {
|
|
msgp.Encodable
|
|
msgp.Marshaler
|
|
msgp.Decodable
|
|
msgp.Unmarshaler
|
|
msgp.Sizer
|
|
Add(*T)
|
|
}
|
|
|
|
//msgp:ignore Segmented
|
|
|
|
// Segmented contains f type A metrics segmented by time.
|
|
// FirstTime must be aligned to a start time that it a multiple of Interval.
|
|
type Segmented[T any, PT interface {
|
|
*T
|
|
Segmenter[T]
|
|
}] struct {
|
|
Interval int `json:"intervalSecs,omitempty"` // Interval covered by each segment in seconds.
|
|
FirstTime time.Time `json:"firstTime,omitzero"` // Timestamp of first (ie oldest) segment
|
|
Segments []T `json:"segments,omitempty"` // List of DiskAction for each segment ordered by time (oldest first).
|
|
}
|
|
|
|
// Add 'other' to 'a'.
|
|
func (s *Segmented[T, PT]) Add(other *Segmented[T, PT]) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
if len(other.Segments) == 0 {
|
|
return
|
|
}
|
|
if len(s.Segments) == 0 {
|
|
// Copy slice to avoid overriding the original segment
|
|
*s = *other
|
|
s.Segments = append([]T{}, other.Segments...)
|
|
return
|
|
}
|
|
|
|
// Intervals must match to merge safely.
|
|
if other.Interval == 0 || s.Interval != other.Interval {
|
|
// Cannot merge different resolutions without resampling.
|
|
// Bail out silently as there's no error mechanism here.
|
|
return
|
|
}
|
|
|
|
// Fast-path: same start time and same number of segments -> direct in-place merge.
|
|
if s.FirstTime.Equal(other.FirstTime) && len(s.Segments) == len(other.Segments) {
|
|
for i := range s.Segments {
|
|
t := PT(&s.Segments[i])
|
|
t.Add(&other.Segments[i])
|
|
}
|
|
return
|
|
}
|
|
// More complex merge...
|
|
step := time.Duration(s.Interval) * time.Second
|
|
|
|
// Determine the unified timeline.
|
|
start := s.FirstTime
|
|
if other.FirstTime.Before(start) {
|
|
start = other.FirstTime
|
|
}
|
|
|
|
// Compute end times (exclusive).
|
|
aEnd := s.FirstTime.Add(time.Duration(len(s.Segments)) * step)
|
|
oEnd := other.FirstTime.Add(time.Duration(len(other.Segments)) * step)
|
|
|
|
// Total number of slots to cover both series.
|
|
totalSlots := int(oEnd.Sub(start) / step)
|
|
if aEnd.After(oEnd) {
|
|
totalSlots = int(aEnd.Sub(start) / step)
|
|
}
|
|
|
|
// Prepare the result slice with zero-value APIStats (acts as empty).
|
|
newSegments := make([]T, totalSlots)
|
|
|
|
// Copy/merge 's' into new slice at the proper offset.
|
|
if s.FirstTime.After(start) {
|
|
offset := int(s.FirstTime.Sub(start) / step)
|
|
copy(newSegments[offset:offset+len(s.Segments)], s.Segments)
|
|
} else {
|
|
// s starts at 'start'
|
|
copy(newSegments[:len(s.Segments)], s.Segments)
|
|
}
|
|
|
|
// Merge 'other' into new slice at the proper offset.
|
|
otherOffset := int(other.FirstTime.Sub(start) / step)
|
|
for i, s := range other.Segments {
|
|
idx := otherOffset + i
|
|
if idx < 0 || idx >= len(newSegments) {
|
|
continue
|
|
}
|
|
pt := PT(&newSegments[idx])
|
|
pt.Add(&s)
|
|
}
|
|
|
|
// Update receiver with merged result.
|
|
s.FirstTime = start
|
|
s.Segments = newSegments
|
|
}
|
|
|
|
// Total returns the total of all segments.
|
|
func (s *Segmented[T, PT]) Total() T {
|
|
var res T
|
|
if s == nil {
|
|
return res
|
|
}
|
|
pt := PT(&res)
|
|
for i := range s.Segments {
|
|
pt.Add(&s.Segments[i])
|
|
}
|
|
// Since we are merging across APIs must reset track node count.
|
|
return res
|
|
}
|
|
|
|
// ReplicationMetrics contains metrics for outgoing replication operations.
|
|
type ReplicationMetrics struct {
|
|
// Time these metrics were collected
|
|
CollectedAt time.Time `json:"collected"`
|
|
|
|
// Nodes responded to the request.
|
|
Nodes int `json:"nodes"`
|
|
|
|
// Number of active replication events.
|
|
Active int64 `json:"active,omitempty"`
|
|
|
|
// Number of queued replication events.
|
|
Queued int64 `json:"queued,omitempty"`
|
|
|
|
Targets map[string]ReplicationTargetStats `json:"targets"`
|
|
|
|
// Received tracks aggregate inbound replication stats across all sources.
|
|
Received ReplicationReceivedStats `json:"received,omitempty"`
|
|
}
|
|
|
|
func (m *ReplicationMetrics) Merge(other *ReplicationMetrics) {
|
|
if m == nil || other == nil {
|
|
return
|
|
}
|
|
if m.CollectedAt.Before(other.CollectedAt) {
|
|
m.CollectedAt = other.CollectedAt
|
|
}
|
|
m.Nodes += other.Nodes
|
|
m.Active += other.Active
|
|
m.Queued += other.Queued
|
|
m.Received.Add(&other.Received)
|
|
|
|
if len(other.Targets) == 0 {
|
|
return
|
|
}
|
|
if m.Targets == nil {
|
|
m.Targets = make(map[string]ReplicationTargetStats, len(other.Targets))
|
|
}
|
|
for k, v := range other.Targets {
|
|
dst := m.Targets[k]
|
|
dst.Merge(&v)
|
|
m.Targets[k] = dst
|
|
}
|
|
}
|
|
|
|
// AllTargets returns aggregated stats for all targets.
|
|
func (m *ReplicationMetrics) AllTargets() ReplicationTargetStats {
|
|
var dst ReplicationTargetStats
|
|
for _, v := range m.Targets {
|
|
dst.Merge(&v)
|
|
}
|
|
return dst
|
|
}
|
|
|
|
// ReplicationTargetStats is replication stats for a single target.
|
|
type ReplicationTargetStats struct {
|
|
// Nodes responded to the request.
|
|
Nodes int `json:"nodes"`
|
|
|
|
// Last minute operation statistics per target.
|
|
LastMinute ReplicationStats `json:"last_minute,omitempty"`
|
|
|
|
// Last hour operation statistics per target.
|
|
LastHour ReplicationStats `json:"last_hour,omitempty"`
|
|
|
|
// Last day operation statistics per target, time segmented.
|
|
LastDay *SegmentedReplicationStats `json:"last_day,omitempty"`
|
|
|
|
// SinceStart contains operations by target.
|
|
SinceStart ReplicationStats `json:"since_start"`
|
|
}
|
|
|
|
// Merge 'other' into 'r'
|
|
func (r *ReplicationTargetStats) Merge(other *ReplicationTargetStats) {
|
|
if r == nil || other == nil || other.Nodes == 0 {
|
|
return
|
|
}
|
|
r.Nodes += other.Nodes
|
|
r.LastMinute.Add(&other.LastMinute)
|
|
r.LastHour.Add(&other.LastHour)
|
|
if r.LastDay == nil && other.LastDay != nil {
|
|
var dst SegmentedReplicationStats
|
|
dst.Add(other.LastDay)
|
|
r.LastDay = &dst
|
|
} else {
|
|
r.LastDay.Add(other.LastDay)
|
|
}
|
|
r.SinceStart.Add(&other.SinceStart)
|
|
}
|
|
|
|
// ReplicationStats is the outgoing replication stats.
|
|
type ReplicationStats struct {
|
|
Nodes int `json:"nodes,omitempty"` // Number of nodes that have reported data.
|
|
StartTime *time.Time `json:"startTime,omitempty"` // Time range this data covers unless merged from sources with different start times..
|
|
EndTime *time.Time `json:"endTime,omitempty"` // Time range this data covers unless merged from sources with different end times.
|
|
WallTimeSecs float64 `json:"wallTimeSecs,omitempty"` // Wall time this data covers, accumulated from all nodes.
|
|
|
|
// Total number of replication events.
|
|
Events int64 `json:"events,omitempty"` // Total number of requests.
|
|
Bytes int64 `json:"bytes,omitempty"` // Total number of bytes sent to remote.
|
|
EventTimeSecs float64 `json:"timeSecs,omitempty"` // Accumulated event time
|
|
|
|
// Latency from queue time to completion.
|
|
LatencySecs float64 `json:"latency,omitempty"` // Accumulated event latency for replication events for all nodes.
|
|
MaxLatencySecs float64 `json:"maxLatency,omitempty"` // Maximum latency for a single node.
|
|
|
|
// Replication event types.
|
|
PutObject int64 `json:"put,omitempty"` // Total put replication requests.
|
|
UpdateMeta int64 `json:"updateMeta,omitempty"` // Total metadata update requests.
|
|
DelObject int64 `json:"del,omitempty"` // Total delete replication requests.
|
|
DelTag int64 `json:"delTag,omitempty"` // Number of DELETE tagging request
|
|
|
|
PutErrors int64 `json:"putErrs,omitempty"` // Replication PutObject event errors.
|
|
UpdateMetaErrors int64 `json:"putTagErrs,omitempty"` // Replication Update Metadata errors.
|
|
DelErrors int64 `json:"delErrs,omitempty"` // Replication DelObject event errors.
|
|
DelTagErrors int64 `json:"delTagErrs,omitempty"` // Replication DelTag event errors.
|
|
|
|
// Outcome (if not error)
|
|
Synced int64 `json:"synced,omitempty"` // Total synced replication requests (didn't exist on remote).
|
|
AlreadyOK int64 `json:"alreadyOK,omitempty"` // Total already-ok replication requests (already existed on remote).
|
|
Rejected int64 `json:"rejected,omitempty"` // Total rejected replication requests.
|
|
|
|
// Proxy to remote counted separately.
|
|
ProxyEvents int64 `json:"proxy,omitempty"` // Number of proxy events.
|
|
ProxyBytes int64 `json:"proxyBytes,omitempty"` // Number of bytes transferred from proxy requests.
|
|
ProxyHead int64 `json:"proxyHead,omitempty"` // Number of HEAD requests proxied to replication target
|
|
ProxyGet int64 `json:"proxyGet,omitempty"` // Number of GET requests proxied to replication target
|
|
ProxyGetTag int64 `json:"proxyGetTag,omitempty"` // Number of GET tagging requests proxied to replication target
|
|
|
|
ProxyHeadOK int64 `json:"proxyHeadOK,omitempty"` // Proxy HEAD requests that were successful.
|
|
ProxyGetOK int64 `json:"proxyGetOK,omitempty"` // Proxy GET requests that were successful.
|
|
ProxyGetTagOK int64 `json:"proxyGetTagOK,omitempty"` // Proxy GET TAG requests that were successful.
|
|
}
|
|
|
|
type SegmentedReplicationStats = Segmented[ReplicationStats, *ReplicationStats]
|
|
|
|
// Add 'other' to a.
|
|
func (a *ReplicationStats) Add(other *ReplicationStats) {
|
|
if other == nil || other.Nodes == 0 {
|
|
return
|
|
}
|
|
// Handle start/end times
|
|
if a.StartTime == nil && a.Events == 0 {
|
|
a.StartTime = other.StartTime
|
|
}
|
|
if a.EndTime == nil && a.Events == 0 {
|
|
a.EndTime = other.EndTime
|
|
}
|
|
if a.StartTime != nil && other.StartTime != nil && !a.StartTime.Equal(*other.StartTime) {
|
|
a.StartTime = nil
|
|
}
|
|
if a.EndTime != nil && other.EndTime != nil && !a.EndTime.Equal(*other.EndTime) {
|
|
a.EndTime = nil
|
|
}
|
|
|
|
// Merge counters
|
|
a.Nodes += other.Nodes
|
|
a.WallTimeSecs += other.WallTimeSecs
|
|
a.Events += other.Events
|
|
a.Bytes += other.Bytes
|
|
a.EventTimeSecs += other.EventTimeSecs
|
|
|
|
// Event types
|
|
a.PutObject += other.PutObject
|
|
a.UpdateMeta += other.UpdateMeta
|
|
a.DelObject += other.DelObject
|
|
a.DelTag += other.DelTag
|
|
|
|
a.LatencySecs += other.LatencySecs
|
|
a.MaxLatencySecs = max(a.MaxLatencySecs, other.MaxLatencySecs)
|
|
|
|
a.PutErrors += other.PutErrors
|
|
a.UpdateMetaErrors += other.UpdateMetaErrors
|
|
a.DelErrors += other.DelErrors
|
|
a.DelTagErrors += other.DelTagErrors
|
|
|
|
// Outcomes
|
|
a.Synced += other.Synced
|
|
a.AlreadyOK += other.AlreadyOK
|
|
a.Rejected += other.Rejected
|
|
|
|
// Proxy events
|
|
a.ProxyEvents += other.ProxyEvents
|
|
a.ProxyBytes += other.ProxyBytes
|
|
a.ProxyHead += other.ProxyHead
|
|
a.ProxyGet += other.ProxyGet
|
|
a.ProxyGetTag += other.ProxyGetTag
|
|
|
|
a.ProxyGetOK += other.ProxyGetOK
|
|
a.ProxyGetTagOK += other.ProxyGetTagOK
|
|
a.ProxyHeadOK += other.ProxyHeadOK
|
|
}
|
|
|
|
// ReceivedStat tracks inbound replication counts and bytes.
|
|
type ReceivedStat struct {
|
|
Count int64 `json:"count"`
|
|
Bytes int64 `json:"bytes"`
|
|
}
|
|
|
|
// Add other into r.
|
|
func (r *ReceivedStat) Add(other *ReceivedStat) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
r.Count += other.Count
|
|
r.Bytes += other.Bytes
|
|
}
|
|
|
|
// ReplicationReceivedStats tracks aggregate inbound replication
|
|
// across time windows.
|
|
type ReplicationReceivedStats struct {
|
|
LastMinute ReceivedStat `json:"lastMinute"`
|
|
LastHour ReceivedStat `json:"lastHour"`
|
|
LastDay ReceivedStat `json:"lastDay"`
|
|
SinceStart ReceivedStat `json:"sinceStart"`
|
|
}
|
|
|
|
// Add other into r.
|
|
func (r *ReplicationReceivedStats) Add(other *ReplicationReceivedStats) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
r.LastMinute.Add(&other.LastMinute)
|
|
r.LastHour.Add(&other.LastHour)
|
|
r.LastDay.Add(&other.LastDay)
|
|
r.SinceStart.Add(&other.SinceStart)
|
|
}
|
|
|
|
// Empty returns true if all windows have zero counts.
|
|
func (r *ReplicationReceivedStats) Empty() bool {
|
|
return r == nil || (r.LastMinute.Count == 0 && r.LastHour.Count == 0 && r.LastDay.Count == 0 && r.SinceStart.Count == 0)
|
|
}
|
|
|
|
// BucketReplWindowedStats holds per-ARN windowed event counts for a single
|
|
// replication direction (failed, retried, or transferred). Bytes are
|
|
// populated for transferred events only; failed/retry events record
|
|
// Count only — Bytes will always be zero for those.
|
|
type BucketReplWindowedStats = ReplicationReceivedStats
|
|
|
|
// ProcessMetrics contains aggregated minio process metrics
|
|
type ProcessMetrics struct {
|
|
CollectedAt time.Time `json:"collected_at,omitempty"`
|
|
Nodes int `json:"nodes,omitempty"`
|
|
|
|
// Aggregated values
|
|
TotalCPUPercent float64 `json:"total_cpu_percent,omitempty"`
|
|
TotalNumConnections int `json:"total_num_connections,omitempty"`
|
|
TotalRunningSecs float64 `json:"total_running_secs,omitempty"`
|
|
TotalNumFDs int64 `json:"total_num_fds,omitempty"`
|
|
TotalNumThreads int64 `json:"total_num_threads,omitempty"`
|
|
TotalNice int64 `json:"total_nice,omitempty"`
|
|
Count int `json:"count,omitempty"`
|
|
|
|
// Counters for boolean fields
|
|
BackgroundProcesses int `json:"background_processes,omitempty"`
|
|
RunningProcesses int `json:"running_processes,omitempty"`
|
|
|
|
// Aggregated memory info
|
|
MemInfo ProcessMemoryInfo `json:"mem_info,omitempty"`
|
|
|
|
// Aggregated IO counters
|
|
IOCounters ProcessIOCounters `json:"io_counters,omitempty"`
|
|
|
|
// Aggregated context switches
|
|
NumCtxSwitches ProcessCtxSwitches `json:"num_ctx_switches,omitempty"`
|
|
|
|
// Aggregated page faults
|
|
PageFaults ProcessPageFaults `json:"page_faults,omitempty"`
|
|
|
|
// Aggregated CPU times
|
|
CPUTimes ProcessCPUTimes `json:"cpu_times,omitempty"`
|
|
|
|
// Aggregated memory maps (platform-specific)
|
|
MemMaps ProcessMemoryMaps `json:"mem_maps,omitempty"`
|
|
|
|
LastDay *SegmentedProcessMetrics `json:"lastDay,omitempty"`
|
|
|
|
// Last hour statistics (1-min segments).
|
|
LastHour *SegmentedProcessMetrics `json:"lastHour,omitempty"`
|
|
}
|
|
|
|
// ProcessMemoryInfo represents aggregated memory information
|
|
type ProcessMemoryInfo struct {
|
|
RSS uint64 `json:"rss,omitempty"`
|
|
VMS uint64 `json:"vms,omitempty"`
|
|
HWM uint64 `json:"hwm,omitempty"`
|
|
Data uint64 `json:"data,omitempty"`
|
|
Stack uint64 `json:"stack,omitempty"`
|
|
Locked uint64 `json:"locked,omitempty"`
|
|
Swap uint64 `json:"swap,omitempty"`
|
|
Count int `json:"count,omitempty"`
|
|
Shared uint64 `json:"shared,omitempty"`
|
|
}
|
|
|
|
// ProcessIOCounters represents aggregated IO counters
|
|
type ProcessIOCounters struct {
|
|
ReadCount uint64 `json:"read_count,omitempty"`
|
|
WriteCount uint64 `json:"write_count,omitempty"`
|
|
ReadBytes uint64 `json:"read_bytes,omitempty"`
|
|
WriteBytes uint64 `json:"write_bytes,omitempty"`
|
|
Count int `json:"count,omitempty"`
|
|
}
|
|
|
|
// ProcessCtxSwitches represents aggregated context switches
|
|
type ProcessCtxSwitches struct {
|
|
Voluntary int64 `json:"voluntary,omitempty"`
|
|
Involuntary int64 `json:"involuntary,omitempty"`
|
|
Count int `json:"count,omitempty"`
|
|
}
|
|
|
|
// ProcessPageFaults represents aggregated page faults
|
|
type ProcessPageFaults struct {
|
|
MinorFaults uint64 `json:"minor_faults,omitempty"`
|
|
MajorFaults uint64 `json:"major_faults,omitempty"`
|
|
ChildMinorFaults uint64 `json:"child_minor_faults,omitempty"`
|
|
ChildMajorFaults uint64 `json:"child_major_faults,omitempty"`
|
|
Count int `json:"count,omitempty"`
|
|
}
|
|
|
|
// ProcessCPUTimes represents aggregated CPU times
|
|
type ProcessCPUTimes struct {
|
|
User float64 `json:"user,omitempty"`
|
|
System float64 `json:"system,omitempty"`
|
|
Idle float64 `json:"idle,omitempty"`
|
|
Nice float64 `json:"nice,omitempty"`
|
|
Iowait float64 `json:"iowait,omitempty"`
|
|
Irq float64 `json:"irq,omitempty"`
|
|
Softirq float64 `json:"softirq,omitempty"`
|
|
Steal float64 `json:"steal,omitempty"`
|
|
Guest float64 `json:"guest,omitempty"`
|
|
GuestNice float64 `json:"guest_nice,omitempty"`
|
|
Count int `json:"count,omitempty"`
|
|
}
|
|
|
|
// ProcessMemoryMaps represents aggregated memory maps (platform-specific)
|
|
type ProcessMemoryMaps struct {
|
|
TotalSize uint64 `json:"total_size,omitempty"`
|
|
TotalRSS uint64 `json:"total_rss,omitempty"`
|
|
TotalPSS uint64 `json:"total_pss,omitempty"`
|
|
TotalSharedClean uint64 `json:"total_shared_clean,omitempty"`
|
|
TotalSharedDirty uint64 `json:"total_shared_dirty,omitempty"`
|
|
TotalPrivateClean uint64 `json:"total_private_clean,omitempty"`
|
|
TotalPrivateDirty uint64 `json:"total_private_dirty,omitempty"`
|
|
TotalReferenced uint64 `json:"total_referenced,omitempty"`
|
|
TotalAnonymous uint64 `json:"total_anonymous,omitempty"`
|
|
TotalSwap uint64 `json:"total_swap,omitempty"`
|
|
Count int `json:"count,omitempty"`
|
|
}
|
|
|
|
// Merge merges process metrics from another ProcessMetrics
|
|
func (m *ProcessMetrics) Merge(other *ProcessMetrics) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
|
|
// Update timestamp to the latest
|
|
if other.CollectedAt.After(m.CollectedAt) {
|
|
m.CollectedAt = other.CollectedAt
|
|
}
|
|
|
|
m.Nodes += other.Nodes
|
|
m.TotalCPUPercent += other.TotalCPUPercent
|
|
m.TotalNumConnections += other.TotalNumConnections
|
|
m.TotalRunningSecs += other.TotalRunningSecs
|
|
m.TotalNumFDs += other.TotalNumFDs
|
|
m.TotalNumThreads += other.TotalNumThreads
|
|
m.TotalNice += other.TotalNice
|
|
m.Count += other.Count
|
|
|
|
// Merge boolean counters
|
|
m.BackgroundProcesses += other.BackgroundProcesses
|
|
m.RunningProcesses += other.RunningProcesses
|
|
|
|
// Merge memory info
|
|
m.MemInfo.RSS += other.MemInfo.RSS
|
|
m.MemInfo.VMS += other.MemInfo.VMS
|
|
m.MemInfo.HWM += other.MemInfo.HWM
|
|
m.MemInfo.Data += other.MemInfo.Data
|
|
m.MemInfo.Stack += other.MemInfo.Stack
|
|
m.MemInfo.Locked += other.MemInfo.Locked
|
|
m.MemInfo.Swap += other.MemInfo.Swap
|
|
m.MemInfo.Count += other.MemInfo.Count
|
|
m.MemInfo.Shared += other.MemInfo.Shared
|
|
|
|
// Merge IO counters
|
|
m.IOCounters.ReadCount += other.IOCounters.ReadCount
|
|
m.IOCounters.WriteCount += other.IOCounters.WriteCount
|
|
m.IOCounters.ReadBytes += other.IOCounters.ReadBytes
|
|
m.IOCounters.WriteBytes += other.IOCounters.WriteBytes
|
|
m.IOCounters.Count += other.IOCounters.Count
|
|
|
|
// Merge context switches
|
|
m.NumCtxSwitches.Voluntary += other.NumCtxSwitches.Voluntary
|
|
m.NumCtxSwitches.Involuntary += other.NumCtxSwitches.Involuntary
|
|
m.NumCtxSwitches.Count += other.NumCtxSwitches.Count
|
|
|
|
// Merge page faults
|
|
m.PageFaults.MinorFaults += other.PageFaults.MinorFaults
|
|
m.PageFaults.MajorFaults += other.PageFaults.MajorFaults
|
|
m.PageFaults.ChildMinorFaults += other.PageFaults.ChildMinorFaults
|
|
m.PageFaults.ChildMajorFaults += other.PageFaults.ChildMajorFaults
|
|
m.PageFaults.Count += other.PageFaults.Count
|
|
|
|
// Merge CPU times
|
|
m.CPUTimes.User += other.CPUTimes.User
|
|
m.CPUTimes.System += other.CPUTimes.System
|
|
m.CPUTimes.Idle += other.CPUTimes.Idle
|
|
m.CPUTimes.Nice += other.CPUTimes.Nice
|
|
m.CPUTimes.Iowait += other.CPUTimes.Iowait
|
|
m.CPUTimes.Irq += other.CPUTimes.Irq
|
|
m.CPUTimes.Softirq += other.CPUTimes.Softirq
|
|
m.CPUTimes.Steal += other.CPUTimes.Steal
|
|
m.CPUTimes.Guest += other.CPUTimes.Guest
|
|
m.CPUTimes.GuestNice += other.CPUTimes.GuestNice
|
|
m.CPUTimes.Count += other.CPUTimes.Count
|
|
|
|
// Merge memory maps
|
|
m.MemMaps.TotalSize += other.MemMaps.TotalSize
|
|
m.MemMaps.TotalRSS += other.MemMaps.TotalRSS
|
|
m.MemMaps.TotalPSS += other.MemMaps.TotalPSS
|
|
m.MemMaps.TotalSharedClean += other.MemMaps.TotalSharedClean
|
|
m.MemMaps.TotalSharedDirty += other.MemMaps.TotalSharedDirty
|
|
m.MemMaps.TotalPrivateClean += other.MemMaps.TotalPrivateClean
|
|
m.MemMaps.TotalPrivateDirty += other.MemMaps.TotalPrivateDirty
|
|
m.MemMaps.TotalReferenced += other.MemMaps.TotalReferenced
|
|
m.MemMaps.TotalAnonymous += other.MemMaps.TotalAnonymous
|
|
m.MemMaps.TotalSwap += other.MemMaps.TotalSwap
|
|
m.MemMaps.Count += other.MemMaps.Count
|
|
if other.LastDay != nil {
|
|
if m.LastDay == nil {
|
|
m.LastDay = new(SegmentedProcessMetrics)
|
|
}
|
|
m.LastDay.Add(other.LastDay)
|
|
}
|
|
if other.LastHour != nil {
|
|
if m.LastHour == nil {
|
|
m.LastHour = new(SegmentedProcessMetrics)
|
|
}
|
|
m.LastHour.Add(other.LastHour)
|
|
}
|
|
}
|
|
|
|
// ProcessSegment contains compact process metrics for time-series segmentation.
|
|
type ProcessSegment struct {
|
|
CPUPercent float64 `json:"cpu_percent,omitempty"`
|
|
NumConnections int `json:"num_connections,omitempty"`
|
|
NumFDs int64 `json:"num_fds,omitempty"`
|
|
NumThreads int64 `json:"num_threads,omitempty"`
|
|
ReadCount uint64 `json:"read_count,omitempty"`
|
|
WriteCount uint64 `json:"write_count,omitempty"`
|
|
ReadBytes uint64 `json:"read_bytes,omitempty"`
|
|
WriteBytes uint64 `json:"write_bytes,omitempty"`
|
|
|
|
RSS uint64 `json:"rss,omitempty"`
|
|
VMS uint64 `json:"vms,omitempty"`
|
|
|
|
CtxSwitchesVoluntary int64 `json:"ctx_switches_voluntary,omitempty"`
|
|
CtxSwitchesInvoluntary int64 `json:"ctx_switches_involuntary,omitempty"`
|
|
|
|
MinorFaults uint64 `json:"minor_faults,omitempty"`
|
|
MajorFaults uint64 `json:"major_faults,omitempty"`
|
|
|
|
// CPU time in seconds
|
|
CPUUser float64 `json:"cpu_user,omitempty"`
|
|
CPUSystem float64 `json:"cpu_system,omitempty"`
|
|
CPUIdle float64 `json:"cpu_idle,omitempty"`
|
|
CPUNice float64 `json:"cpu_nice,omitempty"`
|
|
CPUIowait float64 `json:"cpu_iowait,omitempty"`
|
|
CPUIrq float64 `json:"cpu_irq,omitempty"`
|
|
CPUSoftirq float64 `json:"cpu_softirq,omitempty"`
|
|
CPUSteal float64 `json:"cpu_steal,omitempty"`
|
|
CPUGuest float64 `json:"cpu_guest,omitempty"`
|
|
CPUGuestNice float64 `json:"cpu_guest_nice,omitempty"`
|
|
|
|
N int `json:"n"`
|
|
}
|
|
|
|
// Add other to p for Segmenter interface.
|
|
func (p *ProcessSegment) Add(other *ProcessSegment) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
p.CPUPercent += other.CPUPercent
|
|
p.NumConnections += other.NumConnections
|
|
p.NumFDs += other.NumFDs
|
|
p.NumThreads += other.NumThreads
|
|
p.ReadCount += other.ReadCount
|
|
p.WriteCount += other.WriteCount
|
|
p.ReadBytes += other.ReadBytes
|
|
p.WriteBytes += other.WriteBytes
|
|
p.RSS += other.RSS
|
|
p.VMS += other.VMS
|
|
p.CtxSwitchesVoluntary += other.CtxSwitchesVoluntary
|
|
p.CtxSwitchesInvoluntary += other.CtxSwitchesInvoluntary
|
|
p.MinorFaults += other.MinorFaults
|
|
p.MajorFaults += other.MajorFaults
|
|
p.CPUUser += other.CPUUser
|
|
p.CPUSystem += other.CPUSystem
|
|
p.CPUIdle += other.CPUIdle
|
|
p.CPUNice += other.CPUNice
|
|
p.CPUIowait += other.CPUIowait
|
|
p.CPUIrq += other.CPUIrq
|
|
p.CPUSoftirq += other.CPUSoftirq
|
|
p.CPUSteal += other.CPUSteal
|
|
p.CPUGuest += other.CPUGuest
|
|
p.CPUGuestNice += other.CPUGuestNice
|
|
p.N += other.N
|
|
}
|
|
|
|
// SegmentedProcessMetrics are time-segmented process metrics.
|
|
type SegmentedProcessMetrics = Segmented[ProcessSegment, *ProcessSegment]
|
|
|
|
// HealOrigin identifies the subsystem that triggered a healing operation.
|
|
type HealOrigin = string
|
|
|
|
const (
|
|
HealOriginScanner HealOrigin = "scanner"
|
|
HealOriginReadRepair HealOrigin = "read-repair"
|
|
HealOriginDiskReplace HealOrigin = "disk-replace"
|
|
HealOriginDiskOffline HealOrigin = "disk-offline"
|
|
HealOriginManual HealOrigin = "manual"
|
|
HealOriginCrossPool HealOrigin = "cross-pool"
|
|
)
|
|
|
|
// HealError classifies healing failure reasons.
|
|
type HealError = string
|
|
|
|
const (
|
|
HealErrCorrupt HealError = "corrupt"
|
|
HealErrMissing HealError = "missing"
|
|
HealErrOffline HealError = "offline"
|
|
HealErrTimeout HealError = "timeout"
|
|
HealErrPermission HealError = "permission"
|
|
HealErrChecksum HealError = "checksum"
|
|
HealErrReadQuorum HealError = "read-quorum"
|
|
HealErrWriteQuorum HealError = "write-quorum"
|
|
HealErrWarmTierUnreach HealError = "warm-tier-unreachable"
|
|
HealErrWarmTierMissing HealError = "warm-tier-missing"
|
|
)
|
|
|
|
// HealingCounts contains aggregate healing counters.
|
|
// Also serves as the segment type for SegmentedHealingStats.
|
|
type HealingCounts struct {
|
|
Started int64 `json:"started,omitempty"`
|
|
Completed int64 `json:"completed,omitempty"`
|
|
Failed int64 `json:"failed,omitempty"`
|
|
|
|
// Healed is the subset of Completed where drives were actually repaired.
|
|
Healed int64 `json:"healed,omitempty"`
|
|
|
|
// BytesHealed is the total size of objects where drives were actually repaired.
|
|
BytesHealed int64 `json:"bytes_healed,omitempty"`
|
|
|
|
// Bytes is the total size of all objects submitted for heal checks.
|
|
Bytes int64 `json:"bytes,omitempty"`
|
|
|
|
// BytesCompleted is the total size of objects that completed healing without error.
|
|
BytesCompleted int64 `json:"bytes_completed,omitempty"`
|
|
|
|
// AccTime is accumulated wall-clock time of completed heal operations in seconds.
|
|
// Divide by Completed to get average duration.
|
|
AccTime float64 `json:"acc_time_secs,omitempty"`
|
|
|
|
// Dangling is the number of dangling objects detected and cleaned up.
|
|
Dangling int64 `json:"dangling,omitempty"`
|
|
|
|
// WarmTierChecks is the number of warm-tier validation checks performed.
|
|
WarmTierChecks int64 `json:"warm_tier_checks,omitempty"`
|
|
|
|
ByOrigin map[HealOrigin]int64 `json:"by_origin,omitempty"`
|
|
ByType map[HealItemType]int64 `json:"by_type,omitempty"`
|
|
ByError map[HealError]int64 `json:"by_error,omitempty"`
|
|
}
|
|
|
|
// Add other into h. Implements Segmenter[HealingCounts].
|
|
func (h *HealingCounts) Add(other *HealingCounts) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
h.Started += other.Started
|
|
h.Completed += other.Completed
|
|
h.Failed += other.Failed
|
|
h.Healed += other.Healed
|
|
h.BytesHealed += other.BytesHealed
|
|
h.Bytes += other.Bytes
|
|
h.BytesCompleted += other.BytesCompleted
|
|
h.AccTime += other.AccTime
|
|
h.Dangling += other.Dangling
|
|
h.WarmTierChecks += other.WarmTierChecks
|
|
|
|
if len(other.ByOrigin) > 0 {
|
|
if h.ByOrigin == nil {
|
|
h.ByOrigin = make(map[HealOrigin]int64, len(other.ByOrigin))
|
|
}
|
|
for k, v := range other.ByOrigin {
|
|
h.ByOrigin[k] += v
|
|
}
|
|
}
|
|
if len(other.ByType) > 0 {
|
|
if h.ByType == nil {
|
|
h.ByType = make(map[HealItemType]int64, len(other.ByType))
|
|
}
|
|
for k, v := range other.ByType {
|
|
h.ByType[k] += v
|
|
}
|
|
}
|
|
if len(other.ByError) > 0 {
|
|
if h.ByError == nil {
|
|
h.ByError = make(map[HealError]int64, len(other.ByError))
|
|
}
|
|
for k, v := range other.ByError {
|
|
h.ByError[k] += v
|
|
}
|
|
}
|
|
}
|
|
|
|
// SegmentedHealingStats are time-segmented healing metrics.
|
|
type SegmentedHealingStats = Segmented[HealingCounts, *HealingCounts]
|
|
|
|
// HealBucketStats tracks healing outcomes for a single bucket.
|
|
type HealBucketStats struct {
|
|
Started int64 `json:"started,omitempty"`
|
|
Completed int64 `json:"completed,omitempty"`
|
|
Failed int64 `json:"failed,omitempty"`
|
|
}
|
|
|
|
// Add other into h.
|
|
func (h *HealBucketStats) Add(other *HealBucketStats) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
h.Started += other.Started
|
|
h.Completed += other.Completed
|
|
h.Failed += other.Failed
|
|
}
|
|
|
|
// HealSession is a snapshot of a single manual heal session.
|
|
type HealSession struct {
|
|
// ClientToken is the routable token (includes node-index suffix in distributed mode).
|
|
ClientToken string `json:"client_token"`
|
|
|
|
// Target scope
|
|
Bucket string `json:"bucket,omitempty"`
|
|
Prefix string `json:"prefix,omitempty"`
|
|
|
|
// Lifecycle
|
|
Status string `json:"status"`
|
|
StartTime time.Time `json:"start_time"`
|
|
EndTime time.Time `json:"end_time,omitempty"`
|
|
|
|
// Settings applied to this session.
|
|
Settings HealOpts `json:"settings"`
|
|
|
|
// Progress counters by item type.
|
|
ScannedItems map[HealItemType]int64 `json:"scanned_items,omitempty"`
|
|
HealedItems map[HealItemType]int64 `json:"healed_items,omitempty"`
|
|
FailedItems map[HealItemType]int64 `json:"failed_items,omitempty"`
|
|
|
|
// LastActivity is the time of the last scan/heal operation.
|
|
LastActivity time.Time `json:"last_activity,omitempty"`
|
|
}
|
|
|
|
// HealingMetrics contains distributed healing metrics across all nodes.
|
|
type HealingMetrics struct {
|
|
CollectedAt time.Time `json:"collected"`
|
|
Nodes int `json:"nodes"`
|
|
|
|
LastMinute HealingCounts `json:"last_minute,omitempty"`
|
|
LastHour HealingCounts `json:"last_hour,omitempty"`
|
|
LastDay *SegmentedHealingStats `json:"last_day,omitempty"`
|
|
SinceStart HealingCounts `json:"since_start,omitempty"`
|
|
|
|
BucketsLastMinute map[string]HealBucketStats `json:"buckets_last_minute,omitempty"`
|
|
BucketsLastHour map[string]HealBucketStats `json:"buckets_last_hour,omitempty"`
|
|
|
|
// ActiveSessions lists manual heal sessions on this node, keyed by clientToken.
|
|
ActiveSessions map[string]HealSession `json:"active_sessions,omitempty"`
|
|
}
|
|
|
|
// Merge other into m.
|
|
func (m *HealingMetrics) Merge(other *HealingMetrics) {
|
|
if m == nil || other == nil {
|
|
return
|
|
}
|
|
if m.CollectedAt.Before(other.CollectedAt) {
|
|
m.CollectedAt = other.CollectedAt
|
|
}
|
|
m.Nodes += other.Nodes
|
|
m.LastMinute.Add(&other.LastMinute)
|
|
m.LastHour.Add(&other.LastHour)
|
|
m.SinceStart.Add(&other.SinceStart)
|
|
|
|
if other.LastDay != nil {
|
|
if m.LastDay == nil {
|
|
m.LastDay = new(SegmentedHealingStats)
|
|
}
|
|
m.LastDay.Add(other.LastDay)
|
|
}
|
|
|
|
if len(other.BucketsLastMinute) > 0 {
|
|
if m.BucketsLastMinute == nil {
|
|
m.BucketsLastMinute = make(map[string]HealBucketStats, len(other.BucketsLastMinute))
|
|
}
|
|
for k, v := range other.BucketsLastMinute {
|
|
dst := m.BucketsLastMinute[k]
|
|
dst.Add(&v)
|
|
m.BucketsLastMinute[k] = dst
|
|
}
|
|
}
|
|
if len(other.BucketsLastHour) > 0 {
|
|
if m.BucketsLastHour == nil {
|
|
m.BucketsLastHour = make(map[string]HealBucketStats, len(other.BucketsLastHour))
|
|
}
|
|
for k, v := range other.BucketsLastHour {
|
|
dst := m.BucketsLastHour[k]
|
|
dst.Add(&v)
|
|
m.BucketsLastHour[k] = dst
|
|
}
|
|
}
|
|
if len(other.ActiveSessions) > 0 {
|
|
if m.ActiveSessions == nil {
|
|
m.ActiveSessions = make(map[string]HealSession, len(other.ActiveSessions))
|
|
}
|
|
// We do not merge entries as we would only expect the same session to be reported from one node.
|
|
maps.Copy(m.ActiveSessions, other.ActiveSessions)
|
|
}
|
|
}
|
|
|
|
// SegmentedKMSActions are time segmented KMS operation stats.
|
|
type SegmentedKMSActions = Segmented[KMSAction, *KMSAction]
|
|
|
|
// KMSRtMetrics contains metrics for KMS operations.
|
|
type KMSRtMetrics struct {
|
|
CollectedAt time.Time `json:"collected"`
|
|
Nodes int `json:"nodes"`
|
|
|
|
NodesOnline int `json:"nodes_online"`
|
|
OnlineSecs float64 `json:"online_secs,omitempty"`
|
|
LastSuccess *time.Time `json:"last_success,omitempty"`
|
|
ActiveOps int64 `json:"active_ops,omitempty"`
|
|
|
|
LastMinute map[string]KMSAction `json:"lastMinute,omitempty"`
|
|
|
|
LastHour map[string]SegmentedKMSActions `json:"lastHour,omitempty"`
|
|
LastDay map[string]SegmentedKMSActions `json:"lastDay,omitempty"`
|
|
}
|
|
|
|
// Merge other into m.
|
|
func (m *KMSRtMetrics) Merge(other *KMSRtMetrics) {
|
|
if m == nil || other == nil {
|
|
return
|
|
}
|
|
if m.CollectedAt.Before(other.CollectedAt) {
|
|
m.CollectedAt = other.CollectedAt
|
|
}
|
|
m.Nodes += other.Nodes
|
|
m.NodesOnline += other.NodesOnline
|
|
m.OnlineSecs = max(m.OnlineSecs, other.OnlineSecs)
|
|
if other.LastSuccess != nil {
|
|
if m.LastSuccess == nil || other.LastSuccess.After(*m.LastSuccess) {
|
|
m.LastSuccess = other.LastSuccess
|
|
}
|
|
}
|
|
m.ActiveOps += other.ActiveOps
|
|
|
|
mergeKMSMap := func(dst *map[string]KMSAction, src map[string]KMSAction) {
|
|
if len(src) == 0 {
|
|
return
|
|
}
|
|
if *dst == nil {
|
|
*dst = make(map[string]KMSAction, len(src))
|
|
}
|
|
for k, v := range src {
|
|
existing := (*dst)[k]
|
|
existing.Add(&v)
|
|
(*dst)[k] = existing
|
|
}
|
|
}
|
|
mergeKMSMap(&m.LastMinute, other.LastMinute)
|
|
|
|
mergeSegMap := func(dst *map[string]SegmentedKMSActions, src map[string]SegmentedKMSActions) {
|
|
if len(src) == 0 {
|
|
return
|
|
}
|
|
if *dst == nil {
|
|
*dst = make(map[string]SegmentedKMSActions, len(src))
|
|
}
|
|
for k, v := range src {
|
|
existing := (*dst)[k]
|
|
existing.Add(&v)
|
|
(*dst)[k] = existing
|
|
}
|
|
}
|
|
mergeSegMap(&m.LastHour, other.LastHour)
|
|
mergeSegMap(&m.LastDay, other.LastDay)
|
|
}
|
|
|
|
// BucketOpStat holds per-operation request counters and byte I/O for one
|
|
// bucket. Bytes are tracked per-operation so byte traffic can be attributed
|
|
// to the specific S3 calls (e.g., GET vs PUT).
|
|
type BucketOpStat struct {
|
|
Requests int64 `json:"requests"`
|
|
Errors4xx int64 `json:"errors4xx,omitempty"`
|
|
Errors5xx int64 `json:"errors5xx,omitempty"`
|
|
BytesIn uint64 `json:"bytesIn,omitempty"`
|
|
BytesOut uint64 `json:"bytesOut,omitempty"`
|
|
}
|
|
|
|
// SegmentedBucketStats holds a time-segmented series for one bucket within a
|
|
// single window (LastHour or LastDay). Slots are ordered oldest-first; index
|
|
// len-1 is the most recent.
|
|
type SegmentedBucketStats struct {
|
|
// IntervalSecs is the duration of each slot in seconds.
|
|
IntervalSecs int `json:"intervalSecs"`
|
|
|
|
// FirstTime is the timestamp of the oldest slot.
|
|
FirstTime time.Time `json:"firstTime"`
|
|
|
|
// Per-category counts; one slot per IntervalSecs.
|
|
Requests []int64 `json:"requests,omitempty"`
|
|
Gets []int64 `json:"gets,omitempty"`
|
|
Puts []int64 `json:"puts,omitempty"`
|
|
Lists []int64 `json:"lists,omitempty"`
|
|
Errors []int64 `json:"errors,omitempty"`
|
|
Errors4xx []int64 `json:"errors4xx,omitempty"`
|
|
Errors5xx []int64 `json:"errors5xx,omitempty"`
|
|
|
|
// BytesIn / BytesOut are per-slot byte counters so callers can both
|
|
// chart byte throughput and sum across slots for a window total.
|
|
BytesIn []int64 `json:"bytesIn,omitempty"`
|
|
BytesOut []int64 `json:"bytesOut,omitempty"`
|
|
}
|
|
|
|
// Merge folds other into s. Slots are right-aligned and summed so the most
|
|
// recent slot always aligns; FirstTime extends to the earliest reported.
|
|
func (s *SegmentedBucketStats) Merge(other *SegmentedBucketStats) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
if s.IntervalSecs == 0 {
|
|
s.IntervalSecs = other.IntervalSecs
|
|
}
|
|
if s.FirstTime.IsZero() || (!other.FirstTime.IsZero() && other.FirstTime.Before(s.FirstTime)) {
|
|
s.FirstTime = other.FirstTime
|
|
}
|
|
s.Requests = addSlices(s.Requests, other.Requests)
|
|
s.Gets = addSlices(s.Gets, other.Gets)
|
|
s.Puts = addSlices(s.Puts, other.Puts)
|
|
s.Lists = addSlices(s.Lists, other.Lists)
|
|
s.Errors = addSlices(s.Errors, other.Errors)
|
|
s.Errors4xx = addSlices(s.Errors4xx, other.Errors4xx)
|
|
s.Errors5xx = addSlices(s.Errors5xx, other.Errors5xx)
|
|
s.BytesIn = addSlices(s.BytesIn, other.BytesIn)
|
|
s.BytesOut = addSlices(s.BytesOut, other.BytesOut)
|
|
}
|
|
|
|
// BucketMetrics holds all data for one bucket across the available time
|
|
// windows. LastMinute is always populated and aggregated per-op (no
|
|
// segments). LastHour and LastDay are populated only when
|
|
// MetricsHourStats / MetricsDayStats are requested, and carry segmented
|
|
// time-series.
|
|
type BucketMetrics struct {
|
|
// LastMinute holds per-S3-operation aggregated stats over the last
|
|
// minute. Always present. Map key is the operation name.
|
|
LastMinute map[string]BucketOpStat `json:"lastMinute,omitempty"`
|
|
|
|
// LastHour holds 1-minute segmented stats over the last hour.
|
|
// Populated only when MetricsHourStats is requested.
|
|
LastHour *SegmentedBucketStats `json:"lastHour,omitempty"`
|
|
|
|
// LastDay holds 15-minute segmented stats over the last day.
|
|
// Populated only when MetricsDayStats is requested.
|
|
LastDay *SegmentedBucketStats `json:"lastDay,omitempty"`
|
|
}
|
|
|
|
// Merge folds other into m. Per-op LastMinute entries are summed; segmented
|
|
// windows are right-aligned and summed.
|
|
func (m *BucketMetrics) Merge(other *BucketMetrics) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
for op, oStat := range other.LastMinute {
|
|
if m.LastMinute == nil {
|
|
m.LastMinute = make(map[string]BucketOpStat, len(other.LastMinute))
|
|
}
|
|
aStat := m.LastMinute[op]
|
|
aStat.Requests += oStat.Requests
|
|
aStat.Errors4xx += oStat.Errors4xx
|
|
aStat.Errors5xx += oStat.Errors5xx
|
|
aStat.BytesIn += oStat.BytesIn
|
|
aStat.BytesOut += oStat.BytesOut
|
|
m.LastMinute[op] = aStat
|
|
}
|
|
if other.LastHour != nil {
|
|
if m.LastHour == nil {
|
|
m.LastHour = &SegmentedBucketStats{}
|
|
}
|
|
m.LastHour.Merge(other.LastHour)
|
|
}
|
|
if other.LastDay != nil {
|
|
if m.LastDay == nil {
|
|
m.LastDay = &SegmentedBucketStats{}
|
|
}
|
|
m.LastDay.Merge(other.LastDay)
|
|
}
|
|
}
|
|
|
|
// BucketAPIMetrics holds per-bucket API statistics. Each bucket's entry
|
|
// carries the populated windows; the windows themselves live on
|
|
// BucketMetrics so a bucket can carry both LastHour and LastDay at once.
|
|
type BucketAPIMetrics struct {
|
|
// N is the number of nodes that reported data.
|
|
N int `json:"n"`
|
|
|
|
// Buckets maps bucket name to its consolidated metrics.
|
|
Buckets map[string]BucketMetrics `json:"buckets,omitempty"`
|
|
}
|
|
|
|
// Merge folds other into b by merging each per-bucket entry.
|
|
func (b *BucketAPIMetrics) Merge(other *BucketAPIMetrics) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
b.N += other.N
|
|
for bucket, ob := range other.Buckets {
|
|
if b.Buckets == nil {
|
|
b.Buckets = make(map[string]BucketMetrics, len(other.Buckets))
|
|
}
|
|
ab := b.Buckets[bucket]
|
|
ab.Merge(&ob)
|
|
b.Buckets[bucket] = ab
|
|
}
|
|
}
|
|
|
|
// TableAPIMetrics holds traffic for all active tables aggregated across nodes.
|
|
type TableAPIMetrics struct {
|
|
// Time these metrics were collected
|
|
CollectedAt time.Time `json:"collected"`
|
|
|
|
// Nodes responding with data
|
|
Nodes int `json:"nodes"`
|
|
|
|
// LastMinute is the aggregate over the last minute across all tables.
|
|
LastMinute *TableAPIStat `json:"lastMinute,omitempty"`
|
|
|
|
// LastHour is the aggregate over the last hour.
|
|
// Populated only when MetricsHourStats is requested.
|
|
LastHour *SegmentedTableIO `json:"lastHour,omitempty"`
|
|
|
|
// LastDay is the aggregate over the last 24 hours.
|
|
// Populated only when MetricsDayStats is requested.
|
|
LastDay *SegmentedTableIO `json:"lastDay,omitempty"`
|
|
|
|
// TopWarehouses contains the top 25 warehouses by request count if MetricsTopWarehouses is set.
|
|
TopWarehouses *TopTableIO `json:"topW,omitempty"`
|
|
|
|
// TopNamespaces contains the top 25 namespaces by request count if MetricsTopNamespaces is set.
|
|
TopNamespaces *TopTableIO `json:"topN,omitempty"`
|
|
|
|
// TopTables contains the top 25 tables by request count if MetricsTopTables is set.
|
|
TopTables *TopTableIO `json:"topT,omitempty"`
|
|
}
|
|
|
|
// TopTableIO provides sorted IO numbers for tables.
|
|
type TopTableIO struct {
|
|
// Minute stats always provided.
|
|
ByRequestsMin []TableIOMetrics `json:"reqMin,omitempty"`
|
|
ByThroughputMin []TableIOMetrics `json:"thrMin,omitempty"`
|
|
|
|
// Hour is provided if MetricsHourStats is set.
|
|
ByRequestsHour []TableIOMetrics `json:"reqHour,omitempty"`
|
|
ByThroughputHour []TableIOMetrics `json:"thrHour,omitempty"`
|
|
|
|
// Day is provided if MetricsDayStats is set.
|
|
ByRequestsDay []TableIOMetrics `json:"reqDay,omitempty"`
|
|
ByThroughputDay []TableIOMetrics `json:"thrDay,omitempty"`
|
|
}
|
|
|
|
// Merge folds other into t. Entries with matching key are summed; the result
|
|
// for each ranked list is sorted by its native ranking (requests or throughput).
|
|
func (t *TopTableIO) Merge(other *TopTableIO, key func(*TableIOMetrics) string) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
t.ByRequestsMin = mergeTopList(t.ByRequestsMin, other.ByRequestsMin, key, cmpByRequests)
|
|
t.ByThroughputMin = mergeTopList(t.ByThroughputMin, other.ByThroughputMin, key, cmpByThroughput)
|
|
t.ByRequestsHour = mergeTopList(t.ByRequestsHour, other.ByRequestsHour, key, cmpByRequests)
|
|
t.ByThroughputHour = mergeTopList(t.ByThroughputHour, other.ByThroughputHour, key, cmpByThroughput)
|
|
t.ByRequestsDay = mergeTopList(t.ByRequestsDay, other.ByRequestsDay, key, cmpByRequests)
|
|
t.ByThroughputDay = mergeTopList(t.ByThroughputDay, other.ByThroughputDay, key, cmpByThroughput)
|
|
}
|
|
|
|
// TopN re-ranks each contained list and trims to n entries. No-op if t is nil
|
|
// or n <= 0.
|
|
func (t *TopTableIO) TopN(n int) {
|
|
if t == nil || n <= 0 {
|
|
return
|
|
}
|
|
t.ByRequestsMin = sortTrimTopList(t.ByRequestsMin, n, cmpByRequests)
|
|
t.ByThroughputMin = sortTrimTopList(t.ByThroughputMin, n, cmpByThroughput)
|
|
t.ByRequestsHour = sortTrimTopList(t.ByRequestsHour, n, cmpByRequests)
|
|
t.ByThroughputHour = sortTrimTopList(t.ByThroughputHour, n, cmpByThroughput)
|
|
t.ByRequestsDay = sortTrimTopList(t.ByRequestsDay, n, cmpByRequests)
|
|
t.ByThroughputDay = sortTrimTopList(t.ByThroughputDay, n, cmpByThroughput)
|
|
}
|
|
|
|
// Merge folds other into t. CollectedAt takes the later timestamp;
|
|
// Nodes is summed; LastMinute is added; LastHour and LastDay are
|
|
// merged segment-wise; Top* groups are merged per-ranking and trimmed
|
|
// back to tableTopTrimTarget once any ranked list grows past tableTopTrimThreshold.
|
|
func (t *TableAPIMetrics) Merge(other *TableAPIMetrics) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
if other.CollectedAt.After(t.CollectedAt) {
|
|
t.CollectedAt = other.CollectedAt
|
|
}
|
|
t.Nodes += other.Nodes
|
|
if other.LastMinute != nil {
|
|
if t.LastMinute == nil {
|
|
t.LastMinute = &TableAPIStat{}
|
|
}
|
|
t.LastMinute.Add(other.LastMinute)
|
|
}
|
|
if other.LastHour != nil {
|
|
if t.LastHour == nil {
|
|
t.LastHour = &SegmentedTableIO{}
|
|
}
|
|
t.LastHour.Merge(other.LastHour)
|
|
}
|
|
if other.LastDay != nil {
|
|
if t.LastDay == nil {
|
|
t.LastDay = &SegmentedTableIO{}
|
|
}
|
|
t.LastDay.Merge(other.LastDay)
|
|
}
|
|
t.TopWarehouses = mergeTopGroup(t.TopWarehouses, other.TopWarehouses, (*TableIOMetrics).KeyWarehouse)
|
|
t.TopNamespaces = mergeTopGroup(t.TopNamespaces, other.TopNamespaces, (*TableIOMetrics).KeyNamespace)
|
|
t.TopTables = mergeTopGroup(t.TopTables, other.TopTables, (*TableIOMetrics).KeyTable)
|
|
}
|
|
|
|
// TopN re-ranks every ranked list in each Top* group and trims to n entries.
|
|
// Pass n <= 0 to leave the lists unchanged.
|
|
func (t *TableAPIMetrics) TopN(n int) {
|
|
if n <= 0 {
|
|
return
|
|
}
|
|
t.TopWarehouses.TopN(n)
|
|
t.TopNamespaces.TopN(n)
|
|
t.TopTables.TopN(n)
|
|
}
|
|
|
|
// mergeTopGroup folds src into dst, allocating dst if needed. Returns dst.
|
|
func mergeTopGroup(dst, src *TopTableIO, key func(*TableIOMetrics) string) *TopTableIO {
|
|
if src == nil {
|
|
return dst
|
|
}
|
|
if dst == nil {
|
|
dst = &TopTableIO{}
|
|
}
|
|
dst.Merge(src, key)
|
|
return dst
|
|
}
|
|
|
|
// TableIOMetrics holds traffic for a table across time windows.
|
|
type TableIOMetrics struct {
|
|
// Table will be populated with table name if only one table.
|
|
Table *string `json:"table"`
|
|
|
|
// Namespace will be populated if all data is from the same namespace
|
|
Namespace *string `json:"namespace"`
|
|
|
|
// Warehouse will be populated if all data is from the same warehouse
|
|
Warehouse *string `json:"warehouse"`
|
|
|
|
TableAPIStat `msg:",flatten"`
|
|
}
|
|
|
|
// Merge sums other into m. Identity fields (Table/Namespace/Warehouse) are
|
|
// copied verbatim on the first non-empty merge and collapsed to nil on
|
|
// disagreement thereafter, preserving the "set iff single value" contract.
|
|
func (m *TableIOMetrics) Merge(other *TableIOMetrics) {
|
|
if other == nil || other.IsZero() {
|
|
return
|
|
}
|
|
if m.IsZero() {
|
|
m.Table = other.Table
|
|
m.Namespace = other.Namespace
|
|
m.Warehouse = other.Warehouse
|
|
} else {
|
|
m.Table = mergeIdentityField(m.Table, other.Table)
|
|
m.Namespace = mergeIdentityField(m.Namespace, other.Namespace)
|
|
m.Warehouse = mergeIdentityField(m.Warehouse, other.Warehouse)
|
|
}
|
|
m.Add(&other.TableAPIStat)
|
|
}
|
|
|
|
// mergeIdentityField returns a when both pointers reference the same value;
|
|
// nil on any disagreement, including when either side is nil (a nil side
|
|
// already represents "multiple values").
|
|
func mergeIdentityField(a, b *string) *string {
|
|
if a == nil || b == nil {
|
|
return nil
|
|
}
|
|
if *a != *b {
|
|
return nil
|
|
}
|
|
return a
|
|
}
|
|
|
|
type SegmentedTableIO struct {
|
|
// IntervalSecs is the duration of each slot in seconds.
|
|
IntervalSecs int `json:"intervalSecs"`
|
|
|
|
// FirstTime is the timestamp of the oldest slot.
|
|
FirstTime time.Time `json:"firstTime"`
|
|
|
|
// Per-category counts; one slot per IntervalSecs.
|
|
Reads []int64 `json:"reads,omitempty"`
|
|
Writes []int64 `json:"writes,omitempty"`
|
|
BytesIn []int64 `json:"bytesIn,omitempty"`
|
|
BytesOut []int64 `json:"bytesOut,omitempty"`
|
|
NotOK []int64 `json:"notOk,omitempty"`
|
|
RequestTimeSecs []float32 `json:"timeSecs,omitempty"`
|
|
RespTTFBSecs []float32 `json:"ttfbSecs,omitempty"`
|
|
}
|
|
|
|
// Merge folds other into s. Slots are right-aligned and summed so the most
|
|
// recent slot always aligns; FirstTime extends to the earliest reported.
|
|
func (s *SegmentedTableIO) Merge(other *SegmentedTableIO) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
if s.IntervalSecs == 0 {
|
|
s.IntervalSecs = other.IntervalSecs
|
|
}
|
|
if s.FirstTime.IsZero() || (!other.FirstTime.IsZero() && other.FirstTime.Before(s.FirstTime)) {
|
|
s.FirstTime = other.FirstTime
|
|
}
|
|
s.Reads = addSlices(s.Reads, other.Reads)
|
|
s.Writes = addSlices(s.Writes, other.Writes)
|
|
s.BytesIn = addSlices(s.BytesIn, other.BytesIn)
|
|
s.BytesOut = addSlices(s.BytesOut, other.BytesOut)
|
|
s.NotOK = addSlices(s.NotOK, other.NotOK)
|
|
s.RequestTimeSecs = addSlices(s.RequestTimeSecs, other.RequestTimeSecs)
|
|
s.RespTTFBSecs = addSlices(s.RespTTFBSecs, other.RespTTFBSecs)
|
|
}
|
|
|
|
// AsTableIOStat returns one TableAPIStat per slot, right-aligned so the most
|
|
// recent slot is at the last index. Per-category slices shorter than the
|
|
// longest are aligned to the right (their oldest slots map to leading zeros).
|
|
func (s *SegmentedTableIO) AsTableIOStat() []TableAPIStat {
|
|
if s == nil {
|
|
return nil
|
|
}
|
|
n := len(s.Reads)
|
|
for _, l := range []int{
|
|
len(s.Writes), len(s.BytesIn), len(s.BytesOut), len(s.NotOK),
|
|
len(s.RequestTimeSecs), len(s.RespTTFBSecs),
|
|
} {
|
|
if l > n {
|
|
n = l
|
|
}
|
|
}
|
|
if n == 0 {
|
|
return nil
|
|
}
|
|
res := make([]TableAPIStat, n)
|
|
for i, v := range s.Reads {
|
|
res[n-len(s.Reads)+i].Reads = v
|
|
}
|
|
for i, v := range s.Writes {
|
|
res[n-len(s.Writes)+i].Writes = v
|
|
}
|
|
for i, v := range s.BytesIn {
|
|
res[n-len(s.BytesIn)+i].BytesIn = v
|
|
}
|
|
for i, v := range s.BytesOut {
|
|
res[n-len(s.BytesOut)+i].BytesOut = v
|
|
}
|
|
for i, v := range s.NotOK {
|
|
res[n-len(s.NotOK)+i].NotOK = v
|
|
}
|
|
for i, v := range s.RequestTimeSecs {
|
|
res[n-len(s.RequestTimeSecs)+i].RequestTimeSecs = float64(v)
|
|
}
|
|
for i, v := range s.RespTTFBSecs {
|
|
res[n-len(s.RespTTFBSecs)+i].RespTTFBSecs = float64(v)
|
|
}
|
|
return res
|
|
}
|
|
|
|
// TableAPIStat holds read/write counts and byte totals for a
|
|
// table over one time window.
|
|
// Read/Write is decided by the server based on API type.
|
|
type TableAPIStat struct {
|
|
Reads int64 `json:"r,omitempty"` // Requests classified as reads
|
|
Writes int64 `json:"w,omitempty"` // Requests classified as writes
|
|
BytesIn int64 `json:"in,omitempty"` // Bytes in the Request body
|
|
BytesOut int64 `json:"out,omitempty"` // Bytes in the Response body
|
|
NotOK int64 `json:"err,omitempty"` // Response >= status code 400
|
|
RequestTimeSecs float64 `json:"rSecs,omitempty"` // Total request time in seconds
|
|
RespTTFBSecs float64 `json:"ttfb,omitempty"` // Total time spent on TTFB in seconds(req read -> response first byte) in seconds.
|
|
}
|
|
|
|
// Add sums other into t in place.
|
|
func (t *TableAPIStat) Add(other *TableAPIStat) {
|
|
if other == nil {
|
|
return
|
|
}
|
|
t.Reads += other.Reads
|
|
t.Writes += other.Writes
|
|
t.BytesIn += other.BytesIn
|
|
t.BytesOut += other.BytesOut
|
|
t.RequestTimeSecs += other.RequestTimeSecs
|
|
t.RespTTFBSecs += other.RespTTFBSecs
|
|
t.NotOK += other.NotOK
|
|
}
|
|
|
|
// IsZero reports whether all counters are zero.
|
|
func (t *TableAPIStat) IsZero() bool {
|
|
return t == nil || t.Reads == 0 && t.Writes == 0
|
|
}
|
|
|
|
// Top-list growth bounds for TableAPIMetrics. Merge accumulates entries
|
|
// across sources; once any Top* slice exceeds tableTopTrimThreshold it is
|
|
// re-ranked and clipped back to tableTopTrimTarget. The buffer between the
|
|
// final top-25 surface and tableTopTrimTarget gives later merges room to
|
|
// promote entries that would otherwise be dropped too early.
|
|
const (
|
|
tableTopTrimThreshold = 100
|
|
tableTopTrimTarget = 50
|
|
)
|
|
|
|
// cmpByRequests orders TableIOMetrics descending by total request count
|
|
// (Reads + Writes).
|
|
func cmpByRequests(a, b *TableIOMetrics) int {
|
|
return cmp.Compare(b.Reads+b.Writes, a.Reads+a.Writes)
|
|
}
|
|
|
|
// cmpByThroughput orders TableIOMetrics descending by total bytes
|
|
// (BytesIn + BytesOut).
|
|
func cmpByThroughput(a, b *TableIOMetrics) int {
|
|
return cmp.Compare(b.BytesIn+b.BytesOut, a.BytesIn+a.BytesOut)
|
|
}
|
|
|
|
// mergeTopList unions src into dst, summing entries whose identity key
|
|
// matches and re-trimming back to tableTopTrimTarget once the combined slice
|
|
// crosses tableTopTrimThreshold. Sort order during trim is given by cmpFn.
|
|
func mergeTopList(dst, src []TableIOMetrics, key func(*TableIOMetrics) string, cmpFn func(*TableIOMetrics, *TableIOMetrics) int) []TableIOMetrics {
|
|
if len(src) == 0 {
|
|
return dst
|
|
}
|
|
if cap(dst) < len(dst)+len(src) {
|
|
grown := make([]TableIOMetrics, len(dst), len(dst)+len(src))
|
|
copy(grown, dst)
|
|
dst = grown
|
|
}
|
|
idx := make(map[string]int, len(dst)+len(src))
|
|
for i := range dst {
|
|
idx[key(&dst[i])] = i
|
|
}
|
|
for i := range src {
|
|
k := key(&src[i])
|
|
if pos, ok := idx[k]; ok {
|
|
dst[pos].Merge(&src[i])
|
|
continue
|
|
}
|
|
dst = append(dst, src[i])
|
|
idx[k] = len(dst) - 1
|
|
}
|
|
if len(dst) > tableTopTrimThreshold {
|
|
dst = sortTrimTopList(dst, tableTopTrimTarget, cmpFn)
|
|
}
|
|
return dst
|
|
}
|
|
|
|
// sortTrimTopList sorts a by cmpFn and trims to n entries.
|
|
// Returns a unchanged when n <= 0 or len(a) <= n.
|
|
func sortTrimTopList(a []TableIOMetrics, n int, cmpFn func(*TableIOMetrics, *TableIOMetrics) int) []TableIOMetrics {
|
|
if n <= 0 {
|
|
return a
|
|
}
|
|
slices.SortFunc(a, func(x, y TableIOMetrics) int {
|
|
return cmpFn(&x, &y)
|
|
})
|
|
if len(a) > n {
|
|
a = a[:n]
|
|
}
|
|
return a
|
|
}
|
|
|
|
// Identity keys for Top* slices. ASCII unit separator avoids accidental
|
|
// collisions between user-provided names (which rarely if ever contain it).
|
|
const tableKeySep = "\x1f"
|
|
|
|
func ptrStr(s *string) string {
|
|
if s == nil {
|
|
return ""
|
|
}
|
|
return *s
|
|
}
|
|
|
|
// KeyWarehouse returns the identity key used when merging warehouse-level
|
|
// Top entries.
|
|
func (m *TableIOMetrics) KeyWarehouse() string {
|
|
return ptrStr(m.Warehouse)
|
|
}
|
|
|
|
// KeyNamespace returns the identity key used when merging namespace-level
|
|
// Top entries.
|
|
func (m *TableIOMetrics) KeyNamespace() string {
|
|
return ptrStr(m.Warehouse) + tableKeySep + ptrStr(m.Namespace)
|
|
}
|
|
|
|
// KeyTable returns the identity key used when merging table-level Top
|
|
// entries.
|
|
func (m *TableIOMetrics) KeyTable() string {
|
|
return ptrStr(m.Warehouse) + tableKeySep + ptrStr(m.Namespace) + tableKeySep + ptrStr(m.Table)
|
|
}
|
|
|
|
type addable interface {
|
|
~int | ~int8 | ~int16 | ~int32 | ~int64 |
|
|
~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr |
|
|
~float32 | ~float64
|
|
}
|
|
|
|
// addSlices returns the element-wise sum of a and b, right-aligned so
|
|
// that slot index len-1 (most recent) always corresponds between both.
|
|
// The 'a' array is mutated, but the 'b' value is never mutated.
|
|
func addSlices[A addable](a, b []A) []A {
|
|
if len(b) > len(a) {
|
|
// Extend a and shift values while adding b.
|
|
diff := len(b) - len(a)
|
|
a = append(a, make([]A, diff)...)
|
|
for i := len(a) - 1; i >= diff; i-- {
|
|
a[i] = a[i-diff] + b[i]
|
|
}
|
|
for i := 0; i < diff; i++ {
|
|
a[i] = b[i]
|
|
}
|
|
return a
|
|
}
|
|
offset := len(a) - len(b)
|
|
for i, v := range b {
|
|
a[offset+i] += v
|
|
}
|
|
return a
|
|
}
|