Files
M AlveeandGitHub 47606f6238 distributed job framework (#557)
* feat: add DistJobType enum and sync distjob status types

Adds an iota-based DistJobType enum (mirrors the HealingDriveReason/
OfflineReason pattern already used in heal-commands.go) so distributed
job identity is a typed wire value instead of a bare string, per review
feedback on the eos distributed-job framework PR.

Also syncs DistJobLeaderStatus/DistJobNodeStatus field shapes to match
the current server-side JSON response (JobID/JobType replace JobName;
drops the unused Generation/AssignedSets/top-level CurrentBucket
fields that were never populated by the server).

* fix: bump vulnerable prometheus dep

- Add msgp generation (//msgp:tag json + //go:generate msgp) for
  DistJobLeaderStatus/DistJobNodeStatus, matching the pattern used by
  every other *-commands.go file in this package.
- Add unit tests for ListDistJobStatuses (success path, job-filter
  query param, DistJobTypeUnknown no-filter case, non-200 error path).
- Bump github.com/prometheus/prometheus v0.304.1 -> v0.312.0 to clear
  3 of the 4 govulncheck findings on this branch (GO-2026-5710,
  GO-2026-5381, GO-2026-5264, all fixed by that version). The 4th,
  GO-2026-5662, has no fix published upstream yet and also currently
  fails govulncheck on main independent of this branch.

* fix: pin go:generate directives to go.mod tool versions

Consolidate distjob-commands.go's separate //msgp:tag json directive
into the //go:generate msgp -d "tag json" flag, matching net-stats.go's
style (review feedback, PR #557).

Change every //go:generate msgp / //go:generate stringer directive
(31 occurrences, 30 files) to //go:generate go tool msgp / go tool
stringer, so generation always resolves the exact version pinned in
go.mod's tool block instead of whatever binary happens to be on PATH.
This was the root cause of the earlier stringer-drift CI failure.
Drop the now-unnecessary `go install -v tool` step from lint.yml.
2026-07-01 05:10:23 -04:00

286 lines
7.8 KiB
Go

//
// Copyright (c) 2015-2024 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package madmin
import (
"io"
"net"
"net/http"
"net/url"
"os"
"strings"
"time"
"github.com/minio/minio-go/v7/pkg/s3utils"
)
//msgp:tag json
//go:generate go tool msgp -d clearomitted -d "timezone utc" -file $GOFILE
var (
// MinIO only supports last two versions
// you can force an application by setting env MADMIN_API_VERSION=v3 if you want the
// library to start using v3, which would allow the application to work with MinIO
// servers that are not yet upgraded to v4 Admin API
// AdminAPIVersion - admin api version used in the request.
AdminAPIVersion = func() string {
if v := os.Getenv("MADMIN_API_VERSION"); v == "v3" {
return "v3"
}
return "v4"
}()
// Admin API version prefix
adminAPIPrefix = "/" + AdminAPIVersion
// Admin API version prefix for previous version.
adminAPIOldPrefix = "/v3"
// kmsAPIVersion - is the latest KMS API version, for KMS requests.
// NOTE: MinIO only supports last two versions
kmsAPIVersion = "v1"
// kms API version prefix
kmsAPIPrefix = "/" + kmsAPIVersion
)
// getEndpointURL - construct a new endpoint.
func getEndpointURL(endpoint string, secure bool) (*url.URL, error) {
if strings.Contains(endpoint, ":") {
host, _, err := net.SplitHostPort(endpoint)
if err != nil {
return nil, err
}
if !s3utils.IsValidIP(host) && !s3utils.IsValidDomain(host) {
msg := "Endpoint: " + endpoint + " does not follow ip address or domain name standards."
return nil, ErrInvalidArgument(msg)
}
} else {
if !s3utils.IsValidIP(endpoint) && !s3utils.IsValidDomain(endpoint) {
msg := "Endpoint: " + endpoint + " does not follow ip address or domain name standards."
return nil, ErrInvalidArgument(msg)
}
}
// If secure is false, use 'http' scheme.
scheme := "https"
if !secure {
scheme = "http"
}
// Strip the obvious :443 and :80 from the endpoint
// to avoid the signature mismatch error.
if secure && strings.HasSuffix(endpoint, ":443") {
endpoint = strings.TrimSuffix(endpoint, ":443")
}
if !secure && strings.HasSuffix(endpoint, ":80") {
endpoint = strings.TrimSuffix(endpoint, ":80")
}
// Construct a secured endpoint URL.
endpointURLStr := scheme + "://" + endpoint
endpointURL, err := url.Parse(endpointURLStr)
if err != nil {
return nil, err
}
// Validate incoming endpoint URL.
if err := isValidEndpointURL(endpointURL.String()); err != nil {
return nil, err
}
return endpointURL, nil
}
// Verify if input endpoint URL is valid.
func isValidEndpointURL(endpointURL string) error {
if endpointURL == "" {
return ErrInvalidArgument("Endpoint url cannot be empty.")
}
url, err := url.Parse(endpointURL)
if err != nil {
return ErrInvalidArgument("Endpoint url cannot be parsed.")
}
if url.Path != "/" && url.Path != "" {
return ErrInvalidArgument("Endpoint url cannot have fully qualified paths.")
}
return nil
}
// closeResponse close non nil response with any response Body.
// convenient wrapper to drain any remaining data on response body.
//
// Subsequently this allows golang http RoundTripper
// to re-use the same connection for future requests.
func closeResponse(resp *http.Response) {
// Callers should close resp.Body when done reading from it.
// If resp.Body is not closed, the Client's underlying RoundTripper
// (typically Transport) may not be able to re-use a persistent TCP
// connection to the server for a subsequent "keep-alive" request.
if resp != nil && resp.Body != nil {
// Drain any remaining Body and then close the connection.
// Without this closing connection would disallow re-using
// the same connection for future uses.
// - http://stackoverflow.com/a/17961593/4465767
io.Copy(io.Discard, resp.Body)
resp.Body.Close()
}
}
// TimedAction contains a number of actions and their accumulated duration in nanoseconds.
type TimedAction struct {
Count uint64 `json:"count"`
AccTime uint64 `json:"acc_time_ns"`
MinTime uint64 `json:"min_ns,omitempty"`
MaxTime uint64 `json:"max_ns,omitempty"`
Bytes uint64 `json:"bytes,omitempty"`
}
// Add other to t.
func (t *TimedAction) Add(t2 *TimedAction) {
if t2 != nil {
t.Merge(*t2)
}
}
// Avg returns the average time spent on the action.
func (t TimedAction) Avg() time.Duration {
if t.Count == 0 {
return 0
}
return time.Duration(t.AccTime / t.Count)
}
// AvgBytes returns the average time spent on the action.
func (t TimedAction) AvgBytes() uint64 {
if t.Count == 0 {
return 0
}
return t.Bytes / t.Count
}
// Merge other into t.
func (t *TimedAction) Merge(other TimedAction) {
if t.Count == 0 {
t.MinTime = other.MinTime
}
if other.Count > 0 {
t.MinTime = min(t.MinTime, other.MinTime)
}
t.Count += other.Count
t.AccTime += other.AccTime
t.Bytes += other.Bytes
t.MaxTime = max(t.MaxTime, other.MaxTime)
}
// KMSAction contains per-operation KMS call statistics.
type KMSAction struct {
Count uint64 `json:"n,omitempty"`
AccTime float64 `json:"t,omitempty"`
MinTime float64 `json:"min,omitempty"`
MaxTime float64 `json:"max,omitempty"`
ConnFails uint64 `json:"cf,omitempty"`
RemoteErrs uint64 `json:"re,omitempty"`
}
// Avg returns the average time spent on the action.
func (t KMSAction) Avg() time.Duration {
if t.Count == 0 {
return 0
}
return time.Duration(t.AccTime * float64(time.Second) / float64(t.Count))
}
// Add other to t.
func (t *KMSAction) Add(other *KMSAction) {
if other == nil {
return
}
if t.Count == 0 {
t.MinTime = other.MinTime
}
if other.Count > 0 {
t.MinTime = min(t.MinTime, other.MinTime)
}
t.Count += other.Count
t.AccTime += other.AccTime
t.MaxTime = max(t.MaxTime, other.MaxTime)
t.ConnFails += other.ConnFails
t.RemoteErrs += other.RemoteErrs
}
// DiskAction contains a number of actions and their accumulated duration in nanoseconds.
type DiskAction struct {
// Number of actions performed.
Count uint64 `json:"n,omitempty"`
// Accumulated time spent on the action in seconds.
AccTime float64 `json:"time_s,omitempty"`
// Min and max time spent on the action in seconds.
MinTime float64 `json:"min_s,omitempty"`
MaxTime float64 `json:"max_s,omitempty"`
// Total number of bytes read or written.
Bytes uint64 `json:"b,omitempty"`
// Captures all data availability errors such as
// permission denied, faulty disk and timeout errors.
AvailabilityErrs uint64 `json:"errs_avail,omitempty"`
// Captures all timeouts, if set on individual operations.
Timeouts uint64 `json:"timeouts,omitempty"`
}
// Avg returns the average time spent on the action.
func (t DiskAction) Avg() time.Duration {
if t.Count == 0 {
return 0
}
return time.Duration(t.AccTime * float64(time.Second) / float64(t.Count))
}
// AvgBytes returns the average time spent on the action.
func (t DiskAction) AvgBytes() uint64 {
if t.Count == 0 {
return 0
}
return t.Bytes / t.Count
}
// Add other to t.
func (t *DiskAction) Add(other *DiskAction) {
if other == nil {
return
}
if t.Count == 0 {
t.MinTime = other.MinTime
}
if other.Count > 0 {
t.MinTime = min(t.MinTime, other.MinTime)
}
t.Count += other.Count
t.AccTime += other.AccTime
t.Bytes += other.Bytes
t.MaxTime = max(t.MaxTime, other.MaxTime)
t.AvailabilityErrs += other.AvailabilityErrs
t.Timeouts += other.Timeouts
}