mirror of
https://github.com/hanzos3/madmin-go.git
synced 2026-10-30 20:35:32 +00:00
464 lines
15 KiB
Go
464 lines
15 KiB
Go
//
|
|
// Copyright (c) 2015-2024 MinIO, Inc.
|
|
//
|
|
// This file is part of MinIO Object Storage stack
|
|
//
|
|
// This program is free software: you can redistribute it and/or modify
|
|
// it under the terms of the GNU Affero General Public License as
|
|
// published by the Free Software Foundation, either version 3 of the
|
|
// License, or (at your option) any later version.
|
|
//
|
|
// This program is distributed in the hope that it will be useful,
|
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
// GNU Affero General Public License for more details.
|
|
//
|
|
// You should have received a copy of the GNU Affero General Public License
|
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
//
|
|
|
|
package madmin
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"time"
|
|
)
|
|
|
|
// BatchJobType type to describe batch job types
|
|
type BatchJobType string
|
|
|
|
const (
|
|
BatchJobReplicate BatchJobType = "replicate"
|
|
BatchJobKeyRotate BatchJobType = "keyrotate"
|
|
BatchJobExpire BatchJobType = "expire"
|
|
BatchJobCatalog BatchJobType = "catalog"
|
|
)
|
|
|
|
// SupportedJobTypes supported job types
|
|
var SupportedJobTypes = []BatchJobType{
|
|
BatchJobReplicate,
|
|
BatchJobKeyRotate,
|
|
BatchJobExpire,
|
|
// No need to add new types here, they are added directly in server.
|
|
}
|
|
|
|
// BatchJobReplicateTemplate provides a sample template
|
|
// for batch replication
|
|
const BatchJobReplicateTemplate = `replicate:
|
|
apiVersion: v1
|
|
# source of the objects to be replicated
|
|
source:
|
|
type: TYPE # valid values are "s3" or "minio"
|
|
bucket: BUCKET
|
|
prefix: PREFIX # 'PREFIX' is optional
|
|
# If your source is the 'local' alias specified to 'mc batch start', then the 'endpoint' and 'credentials' fields are optional and can be omitted
|
|
# Either the 'source' or 'remote' *must* be the "local" deployment
|
|
endpoint: "http[s]://HOSTNAME:PORT"
|
|
# path: "on|off|auto" # "on" enables path-style bucket lookup. "off" enables virtual host (DNS)-style bucket lookup. Defaults to "auto"
|
|
credentials:
|
|
accessKey: ACCESS-KEY # Required
|
|
secretKey: SECRET-KEY # Required
|
|
# sessionToken: SESSION-TOKEN # Optional only available when rotating credentials are used
|
|
snowball: # automatically activated if the source is local
|
|
disable: false # optionally turn-off snowball archive transfer
|
|
batch: 100 # upto this many objects per archive
|
|
inmemory: true # indicates if the archive must be staged locally or in-memory
|
|
compress: false # S2/Snappy compressed archive
|
|
smallerThan: 256KiB # create archive for all objects smaller than 256KiB
|
|
skipErrs: false # skips any source side read() errors
|
|
|
|
# target where the objects must be replicated
|
|
target:
|
|
type: TYPE # valid values are "s3" or "minio"
|
|
bucket: BUCKET
|
|
prefix: PREFIX # 'PREFIX' is optional
|
|
# If your source is the 'local' alias specified to 'mc batch start', then the 'endpoint' and 'credentials' fields are optional and can be omitted
|
|
|
|
# Either the 'source' or 'remote' *must* be the "local" deployment
|
|
endpoint: "http[s]://HOSTNAME:PORT"
|
|
# path: "on|off|auto" # "on" enables path-style bucket lookup. "off" enables virtual host (DNS)-style bucket lookup. Defaults to "auto"
|
|
credentials:
|
|
accessKey: ACCESS-KEY
|
|
secretKey: SECRET-KEY
|
|
# sessionToken: SESSION-TOKEN # Optional only available when rotating credentials are used
|
|
|
|
# NOTE: All flags are optional
|
|
# - filtering criteria only applies for all source objects match the criteria
|
|
# - configurable notification endpoints
|
|
# - configurable retries for the job (each retry skips successfully previously replaced objects)
|
|
flags:
|
|
filter:
|
|
newerThan: "7d" # match objects newer than this value (e.g. 7d10h31s)
|
|
olderThan: "7d" # match objects older than this value (e.g. 7d10h31s)
|
|
createdAfter: "date" # match objects created after "date"
|
|
createdBefore: "date" # match objects created before "date"
|
|
|
|
## NOTE: tags are not supported when "source" is remote.
|
|
# tags:
|
|
# - key: "name"
|
|
# value: "pick*" # match objects with tag 'name', with all values starting with 'pick'
|
|
|
|
# metadata:
|
|
# - key: "content-type"
|
|
# value: "image/*" # match objects with 'content-type', with all values starting with 'image/'
|
|
|
|
notify:
|
|
endpoint: "https://notify.endpoint" # notification endpoint to receive job status events
|
|
token: "Bearer xxxxx" # optional authentication token for the notification endpoint
|
|
|
|
retry:
|
|
attempts: 10 # number of retries for the job before giving up
|
|
delay: "500ms" # least amount of delay between each retry
|
|
`
|
|
|
|
// BatchJobKeyRotateTemplate provides a sample template
|
|
// for batch key rotation
|
|
const BatchJobKeyRotateTemplate = `keyrotate:
|
|
apiVersion: v1
|
|
bucket: BUCKET
|
|
prefix: PREFIX
|
|
encryption:
|
|
type: sse-s3 # valid values are sse-s3 and sse-kms
|
|
key: <new-kms-key> # valid only for sse-kms
|
|
context: <new-kms-key-context> # valid only for sse-kms
|
|
|
|
# optional flags based filtering criteria
|
|
# for all objects
|
|
flags:
|
|
filter:
|
|
newerThan: "7d" # match objects newer than this value (e.g. 7d10h31s)
|
|
olderThan: "7d" # match objects older than this value (e.g. 7d10h31s)
|
|
createdAfter: "date" # match objects created after "date"
|
|
createdBefore: "date" # match objects created before "date"
|
|
tags:
|
|
- key: "name"
|
|
value: "pick*" # match objects with tag 'name', with all values starting with 'pick'
|
|
metadata:
|
|
- key: "content-type"
|
|
value: "image/*" # match objects with 'content-type', with all values starting with 'image/'
|
|
kmskey: "key-id" # match objects with KMS key-id (applicable only for sse-kms)
|
|
notify:
|
|
endpoint: "https://notify.endpoint" # notification endpoint to receive job status events
|
|
token: "Bearer xxxxx" # optional authentication token for the notification endpoint
|
|
retry:
|
|
attempts: 10 # number of retries for the job before giving up
|
|
delay: "500ms" # least amount of delay between each retry
|
|
`
|
|
|
|
// BatchJobExpireTemplate provides a sample template
|
|
// for batch expiring objects
|
|
const BatchJobExpireTemplate = `expire:
|
|
apiVersion: v1
|
|
bucket: mybucket # Bucket where this job will expire matching objects from
|
|
prefix: myprefix # (Optional) Prefix under which this job will expire objects matching the rules below.
|
|
rules:
|
|
- type: object # objects with zero ore more older versions
|
|
name: NAME # match object names that satisfy the wildcard expression.
|
|
olderThan: 70h # match objects older than this value
|
|
createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
|
|
tags:
|
|
- key: name
|
|
value: pick* # match objects with tag 'name', all values starting with 'pick'
|
|
metadata:
|
|
- key: content-type
|
|
value: image/* # match objects with 'content-type', all values starting with 'image/'
|
|
size:
|
|
lessThan: 10MiB # match objects with size less than this value (e.g. 10MiB)
|
|
greaterThan: 1MiB # match objects with size greater than this value (e.g. 1MiB)
|
|
purge:
|
|
# retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
|
|
# retainVersions: 5 # keep the latest 5 versions of the object.
|
|
|
|
- type: deleted # objects with delete marker as their latest version
|
|
name: NAME # match object names that satisfy the wildcard expression.
|
|
olderThan: 10h # match objects older than this value (e.g. 7d10h31s)
|
|
createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
|
|
purge:
|
|
# retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
|
|
# retainVersions: 5 # keep the latest 5 versions of the object including delete markers.
|
|
|
|
notify:
|
|
endpoint: https://notify.endpoint # notification endpoint to receive job completion status
|
|
token: Bearer xxxxx # optional authentication token for the notification endpoint
|
|
|
|
retry:
|
|
attempts: 10 # number of retries for the job before giving up
|
|
delay: 500ms # least amount of delay between each retry
|
|
`
|
|
|
|
// BatchJobResult returned by StartBatchJob
|
|
type BatchJobResult struct {
|
|
ID string `json:"id"`
|
|
Type BatchJobType `json:"type"`
|
|
User string `json:"user,omitempty"`
|
|
Started time.Time `json:"started"`
|
|
Elapsed time.Duration `json:"elapsed,omitempty"`
|
|
}
|
|
|
|
// StartBatchJob start a new batch job, input job description is in YAML.
|
|
func (adm *AdminClient) StartBatchJob(ctx context.Context, job string) (BatchJobResult, error) {
|
|
resp, err := adm.executeMethod(ctx, http.MethodPost,
|
|
requestData{
|
|
relPath: adminAPIPrefix + "/start-job",
|
|
content: []byte(job),
|
|
},
|
|
)
|
|
if err != nil {
|
|
return BatchJobResult{}, err
|
|
}
|
|
defer closeResponse(resp)
|
|
if resp.StatusCode != http.StatusOK {
|
|
return BatchJobResult{}, httpRespToErrorResponse(resp)
|
|
}
|
|
|
|
res := BatchJobResult{}
|
|
dec := json.NewDecoder(resp.Body)
|
|
if err = dec.Decode(&res); err != nil {
|
|
return res, err
|
|
}
|
|
|
|
return res, nil
|
|
}
|
|
|
|
// BatchJobStatus contains the last batch job metric
|
|
type BatchJobStatus struct {
|
|
LastMetric JobMetric
|
|
}
|
|
|
|
// BatchJobStatus returns the status of the given job.
|
|
func (adm *AdminClient) BatchJobStatus(ctx context.Context, jobID string) (BatchJobStatus, error) {
|
|
values := make(url.Values)
|
|
values.Set("jobId", jobID)
|
|
|
|
resp, err := adm.executeMethod(ctx, http.MethodGet,
|
|
requestData{
|
|
relPath: adminAPIPrefix + "/status-job",
|
|
queryValues: values,
|
|
},
|
|
)
|
|
if err != nil {
|
|
return BatchJobStatus{}, err
|
|
}
|
|
defer closeResponse(resp)
|
|
if resp.StatusCode != http.StatusOK {
|
|
return BatchJobStatus{}, httpRespToErrorResponse(resp)
|
|
}
|
|
|
|
res := BatchJobStatus{}
|
|
dec := json.NewDecoder(resp.Body)
|
|
if err = dec.Decode(&res); err != nil {
|
|
return res, err
|
|
}
|
|
|
|
return res, nil
|
|
}
|
|
|
|
// DescribeBatchJob - describes a currently running Job.
|
|
func (adm *AdminClient) DescribeBatchJob(ctx context.Context, jobID string) (string, error) {
|
|
values := make(url.Values)
|
|
values.Set("jobId", jobID)
|
|
|
|
resp, err := adm.executeMethod(ctx, http.MethodGet,
|
|
requestData{
|
|
relPath: adminAPIPrefix + "/describe-job",
|
|
queryValues: values,
|
|
},
|
|
)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer closeResponse(resp)
|
|
if resp.StatusCode != http.StatusOK {
|
|
return "", httpRespToErrorResponse(resp)
|
|
}
|
|
|
|
buf, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
return string(buf), nil
|
|
}
|
|
|
|
// GenerateBatchJobOpts is to be implemented in future.
|
|
type GenerateBatchJobOpts struct {
|
|
Type BatchJobType
|
|
}
|
|
|
|
// GenerateBatchJob creates a new job template from standard template
|
|
// TODO: allow configuring yaml values
|
|
func (adm *AdminClient) GenerateBatchJob(_ context.Context, opts GenerateBatchJobOpts) (string, error) {
|
|
// TODO: allow configuring the template to fill values from GenerateBatchJobOpts
|
|
switch opts.Type {
|
|
case BatchJobReplicate:
|
|
return BatchJobReplicateTemplate, nil
|
|
case BatchJobKeyRotate:
|
|
return BatchJobKeyRotateTemplate, nil
|
|
case BatchJobExpire:
|
|
return BatchJobExpireTemplate, nil
|
|
}
|
|
return "", fmt.Errorf("unknown batch job requested: %s", opts.Type)
|
|
}
|
|
|
|
// GetSupportedBatchJobTypes returns the list of server supported batch job
|
|
// types.
|
|
func (adm *AdminClient) GetSupportedBatchJobTypes(ctx context.Context) (supportedTypes []BatchJobType, apiUnavailable bool, err error) {
|
|
resp, err := adm.executeMethod(ctx, http.MethodGet,
|
|
requestData{
|
|
relPath: adminAPIPrefix + "/list-supported-job-types",
|
|
},
|
|
)
|
|
if err != nil {
|
|
return nil, false, err
|
|
}
|
|
defer closeResponse(resp)
|
|
|
|
var buf []byte
|
|
switch resp.StatusCode {
|
|
case http.StatusNotFound, http.StatusUpgradeRequired:
|
|
apiUnavailable = true
|
|
case http.StatusOK:
|
|
if buf, err = io.ReadAll(resp.Body); err == nil {
|
|
err = json.Unmarshal(buf, &supportedTypes)
|
|
}
|
|
default:
|
|
err = httpRespToErrorResponse(resp)
|
|
}
|
|
return supportedTypes, apiUnavailable, err
|
|
}
|
|
|
|
// GenerateBatchJobV2 creates a new job template by requesting the server. This
|
|
// is an EOS only API and returns apiUnavailable=true when API is unsupported.
|
|
func (adm *AdminClient) GenerateBatchJobV2(ctx context.Context, opts GenerateBatchJobOpts) (template string, apiUnavailable bool, err error) {
|
|
if opts.Type == "" {
|
|
err = fmt.Errorf("batch job type is required")
|
|
return "", false, err
|
|
}
|
|
|
|
values := make(url.Values)
|
|
values.Set("jobType", string(opts.Type))
|
|
|
|
resp, err := adm.executeMethod(ctx, http.MethodGet,
|
|
requestData{
|
|
relPath: adminAPIPrefix + "/generate-job",
|
|
queryValues: values,
|
|
},
|
|
)
|
|
if err != nil {
|
|
return "", false, err
|
|
}
|
|
defer closeResponse(resp)
|
|
|
|
var buf []byte
|
|
switch resp.StatusCode {
|
|
case http.StatusNotFound, http.StatusUpgradeRequired:
|
|
apiUnavailable = true
|
|
case http.StatusOK:
|
|
if buf, err = io.ReadAll(resp.Body); err == nil {
|
|
template = string(buf)
|
|
}
|
|
default:
|
|
err = httpRespToErrorResponse(resp)
|
|
}
|
|
return template, apiUnavailable, err
|
|
}
|
|
|
|
// ListBatchJobsResult contains entries for all current jobs.
|
|
type ListBatchJobsResult struct {
|
|
Jobs []BatchJobResult `json:"jobs"`
|
|
}
|
|
|
|
// ListBatchJobsFilter returns list based on following
|
|
// filtering params.
|
|
type ListBatchJobsFilter struct {
|
|
ByJobType string
|
|
}
|
|
|
|
// ListBatchJobs list all the currently active batch jobs
|
|
func (adm *AdminClient) ListBatchJobs(ctx context.Context, fl *ListBatchJobsFilter) (ListBatchJobsResult, error) {
|
|
if fl == nil {
|
|
return ListBatchJobsResult{}, errors.New("ListBatchJobsFilter cannot be nil")
|
|
}
|
|
|
|
values := make(url.Values)
|
|
values.Set("jobType", fl.ByJobType)
|
|
|
|
resp, err := adm.executeMethod(ctx, http.MethodGet,
|
|
requestData{
|
|
relPath: adminAPIPrefix + "/list-jobs",
|
|
queryValues: values,
|
|
},
|
|
)
|
|
if err != nil {
|
|
return ListBatchJobsResult{}, err
|
|
}
|
|
defer closeResponse(resp)
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return ListBatchJobsResult{}, httpRespToErrorResponse(resp)
|
|
}
|
|
|
|
d := json.NewDecoder(resp.Body)
|
|
result := ListBatchJobsResult{}
|
|
if err = d.Decode(&result); err != nil {
|
|
return result, err
|
|
}
|
|
|
|
return result, nil
|
|
}
|
|
|
|
// CancelBatchJob cancels ongoing batch job.
|
|
func (adm *AdminClient) CancelBatchJob(ctx context.Context, jobID string) error {
|
|
values := make(url.Values)
|
|
values.Set("id", jobID)
|
|
|
|
resp, err := adm.executeMethod(ctx, http.MethodDelete,
|
|
requestData{
|
|
relPath: adminAPIPrefix + "/cancel-job",
|
|
queryValues: values,
|
|
},
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer closeResponse(resp)
|
|
if resp.StatusCode != http.StatusNoContent {
|
|
return httpRespToErrorResponse(resp)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// CatalogDataFile contains information about an output file from a catalog job run.
|
|
type CatalogDataFile struct {
|
|
Key string `json:"key"`
|
|
Size uint64 `json:"size"`
|
|
MD5Checksum string `json:"MD5Checksum"`
|
|
}
|
|
|
|
// CatalogManifestVersion represents the version of a catalog manifest.
|
|
type CatalogManifestVersion string
|
|
|
|
// Valid values for CatalogManifestVersion.
|
|
const (
|
|
// We use AWS S3's manifest file version here as we are following the same
|
|
// format at least initially.
|
|
CatalogManifestVersion1 CatalogManifestVersion = "2016-11-30"
|
|
)
|
|
|
|
// CatalogManifest represents the manifest of a catalog job's result.
|
|
type CatalogManifest struct {
|
|
SourceBucket string `json:"sourceBucket"`
|
|
DestinationBucket string `json:"destinationBucket"`
|
|
Version CatalogManifestVersion `json:"version"`
|
|
CreationTimestamp string `json:"creationTimestamp"`
|
|
FileFormat string `json:"fileFormat"`
|
|
FileSchema string `json:"fileSchema"`
|
|
Files []CatalogDataFile `json:"files"`
|
|
}
|