mirror of
https://github.com/hanzoai/s3.git
synced 2026-10-30 14:59:09 +00:00
zooqueen
dd74d3e6d6
hanzoai/s3 was an AGPL-3.0 MinIO fork whose network-copyleft blocks embedding
and reselling as a managed service. Replace it wholesale with a fork of
SeaweedFS (github.com/seaweedfs/seaweedfs, Apache-2.0, Go) which ships a
built-in S3 gateway + Filer.
- License: AGPL-3.0 -> Apache-2.0 (upstream LICENSE kept verbatim, incl.
"Copyright 2025 Chris Lu"); NOTICE added crediting SeaweedFS. No AGPL/BUSL/
SSPL/GPL or enterprise ("ee") files remain.
- Module: go.mod path set to github.com/hanzoai/s3 with a self-replace
(replace github.com/seaweedfs/seaweedfs => ./) so the ~1800 internal Go files
keep their upstream import paths with zero churn. Internal identifiers and
import paths are intentionally NOT renamed.
- CLI rebrand: the binary is built as `s3` (was `weed`); root usage, per-command
help/usage/flag strings, shell autocomplete program name, and the product name
in CLI output now read "Hanzo S3" / `s3`. Users never see `weed`.
(e.g. `s3 server`, `s3 filer`, `s3 master`, `s3 volume`, `s3 s3`, `s3 mount`.)
- Build: `make` (or `go build -o s3 ./weed`) builds the `s3` binary green;
`cmd/weed-db` and `cmd/weed-sql` also build; `go vet ./weed/command/` clean.
- README/LLM.md rewritten for Hanzo S3; AGENTS.md/CLAUDE.md symlinks preserved.
- Dropped a 33MB prebuilt Mach-O `stress` binary upstream had committed; `.gitignore`
updated to ignore the `s3` build output.
243 lines
8.6 KiB
Go
243 lines
8.6 KiB
Go
package erasure_coding
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/admin/config"
|
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
|
|
)
|
|
|
|
// Config extends BaseConfig with erasure coding specific settings
|
|
type Config struct {
|
|
base.BaseConfig
|
|
QuietForSeconds int `json:"quiet_for_seconds"`
|
|
FullnessRatio float64 `json:"fullness_ratio"`
|
|
CollectionFilter string `json:"collection_filter"`
|
|
MinSizeMB int `json:"min_size_mb"`
|
|
PreferredTags []string `json:"preferred_tags"`
|
|
ReplicaPlacement string `json:"replica_placement"` // e.g. "020"; empty falls back to the master default replication
|
|
}
|
|
|
|
// NewDefaultConfig creates a new default erasure coding configuration
|
|
func NewDefaultConfig() *Config {
|
|
return &Config{
|
|
BaseConfig: base.BaseConfig{
|
|
Enabled: true,
|
|
ScanIntervalSeconds: 60 * 60, // 1 hour
|
|
MaxConcurrent: 1,
|
|
},
|
|
QuietForSeconds: 3600, // 1 hour, matching the shell ec.encode -quietFor default
|
|
FullnessRatio: 0.95, // 95%, matching the shell ec.encode -fullPercent default
|
|
CollectionFilter: "",
|
|
MinSizeMB: 30, // 30MB (more reasonable than 100MB)
|
|
PreferredTags: nil,
|
|
}
|
|
}
|
|
|
|
// GetConfigSpec returns the configuration schema for erasure coding tasks
|
|
func GetConfigSpec() base.ConfigSpec {
|
|
return base.ConfigSpec{
|
|
Fields: []*config.Field{
|
|
{
|
|
Name: "enabled",
|
|
JSONName: "enabled",
|
|
Type: config.FieldTypeBool,
|
|
DefaultValue: true,
|
|
Required: false,
|
|
DisplayName: "Enable Erasure Coding Tasks",
|
|
Description: "Whether erasure coding tasks should be automatically created",
|
|
HelpText: "Toggle this to enable or disable automatic erasure coding task generation",
|
|
InputType: "checkbox",
|
|
CSSClasses: "form-check-input",
|
|
},
|
|
{
|
|
Name: "scan_interval_seconds",
|
|
JSONName: "scan_interval_seconds",
|
|
Type: config.FieldTypeInterval,
|
|
DefaultValue: 60 * 60,
|
|
MinValue: 10 * 60,
|
|
MaxValue: 24 * 60 * 60,
|
|
Required: true,
|
|
DisplayName: "Scan Interval",
|
|
Description: "How often to scan for volumes needing erasure coding",
|
|
HelpText: "The system will check for volumes that need erasure coding at this interval",
|
|
Placeholder: "1",
|
|
Unit: config.UnitHours,
|
|
InputType: "interval",
|
|
CSSClasses: "form-control",
|
|
},
|
|
{
|
|
Name: "max_concurrent",
|
|
JSONName: "max_concurrent",
|
|
Type: config.FieldTypeInt,
|
|
DefaultValue: 1,
|
|
MinValue: 1,
|
|
MaxValue: 5,
|
|
Required: true,
|
|
DisplayName: "Max Concurrent Tasks",
|
|
Description: "Maximum number of erasure coding tasks that can run simultaneously",
|
|
HelpText: "Limits the number of erasure coding operations running at the same time",
|
|
Placeholder: "1 (default)",
|
|
Unit: config.UnitCount,
|
|
InputType: "number",
|
|
CSSClasses: "form-control",
|
|
},
|
|
{
|
|
Name: "quiet_for_seconds",
|
|
JSONName: "quiet_for_seconds",
|
|
Type: config.FieldTypeInterval,
|
|
DefaultValue: 3600,
|
|
MinValue: 1,
|
|
MaxValue: 3600,
|
|
Required: true,
|
|
DisplayName: "Quiet Period",
|
|
Description: "Minimum time volume must be quiet before erasure coding",
|
|
HelpText: "Volume must not be modified for this duration before erasure coding",
|
|
Placeholder: "60",
|
|
Unit: config.UnitMinutes,
|
|
InputType: "interval",
|
|
CSSClasses: "form-control",
|
|
},
|
|
{
|
|
Name: "fullness_ratio",
|
|
JSONName: "fullness_ratio",
|
|
Type: config.FieldTypeFloat,
|
|
DefaultValue: 0.95,
|
|
MinValue: 0.0001,
|
|
MaxValue: 1.0,
|
|
Required: true,
|
|
DisplayName: "Fullness Ratio",
|
|
Description: "Minimum fullness ratio to trigger erasure coding",
|
|
HelpText: "Only volumes with this fullness ratio or higher will be erasure coded",
|
|
Placeholder: "0.80 (80%)",
|
|
Unit: config.UnitNone,
|
|
InputType: "number",
|
|
CSSClasses: "form-control",
|
|
},
|
|
{
|
|
Name: "collection_filter",
|
|
JSONName: "collection_filter",
|
|
Type: config.FieldTypeString,
|
|
DefaultValue: "",
|
|
Required: false,
|
|
DisplayName: "Collection Filter",
|
|
Description: "Only process volumes from specific collections",
|
|
HelpText: "Leave empty to process all collections, or specify collection name",
|
|
Placeholder: "my_collection",
|
|
InputType: "text",
|
|
CSSClasses: "form-control",
|
|
},
|
|
{
|
|
Name: "min_size_mb",
|
|
JSONName: "min_size_mb",
|
|
Type: config.FieldTypeInt,
|
|
DefaultValue: 30,
|
|
MinValue: 1,
|
|
MaxValue: 1000,
|
|
Required: true,
|
|
DisplayName: "Minimum Size (MB)",
|
|
Description: "Minimum volume size to consider for erasure coding",
|
|
HelpText: "Only volumes larger than this size will be considered for erasure coding",
|
|
Placeholder: "30",
|
|
Unit: config.UnitNone,
|
|
InputType: "number",
|
|
CSSClasses: "form-control",
|
|
},
|
|
{
|
|
Name: "preferred_tags",
|
|
JSONName: "preferred_tags",
|
|
Type: config.FieldTypeString,
|
|
DefaultValue: "",
|
|
Required: false,
|
|
DisplayName: "Preferred Disk Tags",
|
|
Description: "Comma-separated disk tags to prioritize for EC shard placement",
|
|
HelpText: "EC shards will be placed on disks with these tags first, then fall back to other disks if needed",
|
|
Placeholder: "fast,ssd",
|
|
InputType: "text",
|
|
CSSClasses: "form-control",
|
|
},
|
|
{
|
|
Name: "replica_placement",
|
|
JSONName: "replica_placement",
|
|
Type: config.FieldTypeString,
|
|
DefaultValue: "",
|
|
Required: false,
|
|
DisplayName: "Replica Placement",
|
|
Description: "EC shard replica placement constraint (e.g. 020)",
|
|
HelpText: "Leave empty to use the master default replication. When set, the 2nd/3rd digits cap EC shards per rack and per node (best-effort during encode: relaxed rather than failing if the cluster can't satisfy them, then enforced by rebalancing). The 1st (data-center) digit is ignored for EC placement",
|
|
Placeholder: "020",
|
|
InputType: "text",
|
|
CSSClasses: "form-control",
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// ToTaskPolicy converts configuration to a TaskPolicy protobuf message
|
|
func (c *Config) ToTaskPolicy() *worker_pb.TaskPolicy {
|
|
// Defensive copy of PreferredTags to prevent external mutation
|
|
preferredTagsCopy := append([]string(nil), c.PreferredTags...)
|
|
return &worker_pb.TaskPolicy{
|
|
Enabled: c.Enabled,
|
|
MaxConcurrent: int32(c.MaxConcurrent),
|
|
RepeatIntervalSeconds: int32(c.ScanIntervalSeconds),
|
|
CheckIntervalSeconds: int32(c.ScanIntervalSeconds),
|
|
TaskConfig: &worker_pb.TaskPolicy_ErasureCodingConfig{
|
|
ErasureCodingConfig: &worker_pb.ErasureCodingTaskConfig{
|
|
FullnessRatio: float64(c.FullnessRatio),
|
|
QuietForSeconds: int32(c.QuietForSeconds),
|
|
MinVolumeSizeMb: int32(c.MinSizeMB),
|
|
CollectionFilter: c.CollectionFilter,
|
|
PreferredTags: preferredTagsCopy,
|
|
ReplicaPlacement: c.ReplicaPlacement,
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
// FromTaskPolicy loads configuration from a TaskPolicy protobuf message
|
|
func (c *Config) FromTaskPolicy(policy *worker_pb.TaskPolicy) error {
|
|
if policy == nil {
|
|
return fmt.Errorf("policy is nil")
|
|
}
|
|
|
|
// Set general TaskPolicy fields
|
|
c.Enabled = policy.Enabled
|
|
c.MaxConcurrent = int(policy.MaxConcurrent)
|
|
c.ScanIntervalSeconds = int(policy.RepeatIntervalSeconds) // Direct seconds-to-seconds mapping
|
|
|
|
// Set erasure coding-specific fields from the task config
|
|
if ecConfig := policy.GetErasureCodingConfig(); ecConfig != nil {
|
|
c.FullnessRatio = float64(ecConfig.FullnessRatio)
|
|
c.QuietForSeconds = int(ecConfig.QuietForSeconds)
|
|
c.MinSizeMB = int(ecConfig.MinVolumeSizeMb)
|
|
c.CollectionFilter = ecConfig.CollectionFilter
|
|
c.PreferredTags = append([]string(nil), ecConfig.PreferredTags...)
|
|
c.ReplicaPlacement = ecConfig.ReplicaPlacement
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// LoadConfigFromPersistence loads configuration from the persistence layer if available
|
|
func LoadConfigFromPersistence(configPersistence interface{}) *Config {
|
|
config := NewDefaultConfig()
|
|
|
|
// Try to load from persistence if available
|
|
if persistence, ok := configPersistence.(interface {
|
|
LoadErasureCodingTaskPolicy() (*worker_pb.TaskPolicy, error)
|
|
}); ok {
|
|
if policy, err := persistence.LoadErasureCodingTaskPolicy(); err == nil && policy != nil {
|
|
if err := config.FromTaskPolicy(policy); err == nil {
|
|
glog.V(1).Infof("Loaded erasure coding configuration from persistence")
|
|
return config
|
|
}
|
|
}
|
|
}
|
|
|
|
glog.V(1).Infof("Using default erasure coding configuration")
|
|
return config
|
|
}
|