mirror of
https://github.com/hanzoai/s3-csi.git
synced 2026-09-30 23:52:32 +00:00
e8b89e78c3
* fix(docker): clone the fork that exists, at a commit that exists
The s3-mount image has never built. A global rename turned the upstream
seaweedfs reference into `github.com/s3/s3`, which is not a repository, so the
build died on `git clone ... exit code 128` before compiling anything.
Three more things were wrong behind it, each of which would have failed the next
step in turn:
clone github.com/s3/s3 -> github.com/hanzoai/s3
path /go/src/github.com/s3/s3 -> /go/src/github.com/hanzoai/s3
dir cd weed -> cd s3 (the fork renamed it)
commit 19ce7c0b6fdb -> 6dda4e4d6bca
That commit is the real one. 19ce7c0b6fdb is an UPSTREAM seaweedfs hash and does
not exist in our fork — the API 422s on it — so the checkout could never have
succeeded even with the URL fixed. The ARG stays so a build can still pin.
SEAWEEDFS_COMMIT is renamed S3_COMMIT while we are here; nothing consumed it yet.
* chore: no seaweed, no weed — it is s3
369 occurrences across 33 files, plus nine files whose names carried it. All of
it gone: manifests, helm chart, nomad jobs, workflows, docs and code.
Two of these were not cosmetic.
The driver spawned a binary called "weed":
DefaultWeedBinary = "weed"
Our fork builds it as `s3` — the running filer pod has /usr/bin/s3 and no weed
at all — so every mount would have failed with an executable that is not there,
however well the image built.
And the crash-recovery integration test killed processes matching `pgrep -x
weed`. Nothing is named that any more, so it would have found no pids, killed
nothing, and passed while testing exactly nothing. It now matches `s3`. The test
already had a guard that fails when there is nothing to kill, which is the only
reason this would have been noticed at all.
The rest follows the same rename the Dockerfile got: SEAWEEDFS_COMMIT is
S3_COMMIT, and the release/sync workflows that patch that ARG were updated with
it or they would silently stop pinning anything.
No upstream import paths were touched — go.mod depends on github.com/hanzoai/s3,
not on anything upstream, so nothing here was load-bearing for the build.
Verified: go build clean, all three test packages pass.
---------
Co-authored-by: hanzo-dev <[email protected]>
Co-authored-by: Hanzo Dev <[email protected]>
318 lines
6.5 KiB
HCL
318 lines
6.5 KiB
HCL
# WWW naster: http://s3-master.service.consul:9333/
|
|
# WWW filer: http://s3-filer.service.consul:8888/
|
|
|
|
job "s3" {
|
|
datacenters = ["dc1"]
|
|
type = "service"
|
|
|
|
group "s3-master" {
|
|
count = 3
|
|
|
|
constraint {
|
|
attribute = "${attr.unique.hostname}"
|
|
operator = "regexp"
|
|
# We need static IPs for master servers
|
|
# dc1-n1 - 172.21.100.51
|
|
# dc1-n2 - 172.21.100.52
|
|
# dc1-n3 - 172.21.100.53
|
|
value = "^dc1-n1|dc1-n2|dc1-n3$"
|
|
}
|
|
|
|
constraint {
|
|
operator = "distinct_hosts"
|
|
value = "true"
|
|
}
|
|
|
|
restart {
|
|
attempts = 10
|
|
interval = "5m"
|
|
delay = "25s"
|
|
mode = "delay"
|
|
}
|
|
|
|
update {
|
|
max_parallel = 1
|
|
stagger = "5m"
|
|
canary = 0
|
|
}
|
|
|
|
migrate {
|
|
min_healthy_time = "2m"
|
|
}
|
|
|
|
network {
|
|
port "http" {
|
|
static = 9333
|
|
}
|
|
port "grpc" {
|
|
static = 19333
|
|
}
|
|
}
|
|
|
|
task "s3-master" {
|
|
driver = "docker"
|
|
env {
|
|
S3_MASTER_VOLUME_GROWTH_COPY_1 = "1"
|
|
S3_MASTER_VOLUME_GROWTH_COPY_2 = "2"
|
|
S3_MASTER_VOLUME_GROWTH_COPY_OTHER = "1"
|
|
}
|
|
config {
|
|
image = "chrislusf/s3:latest"
|
|
force_pull = "true"
|
|
network_mode = "host"
|
|
args = [
|
|
"-v=1", "master",
|
|
"-volumeSizeLimitMB=100",
|
|
"-resumeState=false",
|
|
"-ip=${NOMAD_IP_http}",
|
|
"-port=${NOMAD_PORT_http}",
|
|
"-peers=172.21.100.51:${NOMAD_PORT_http},172.21.100.52:${NOMAD_PORT_http},172.21.100.53:${NOMAD_PORT_http}",
|
|
"-mdir=${NOMAD_TASK_DIR}/master"
|
|
]
|
|
}
|
|
|
|
resources {
|
|
cpu = 128
|
|
memory = 128
|
|
}
|
|
|
|
service {
|
|
tags = ["${node.unique.name}"]
|
|
name = "s3-master"
|
|
port = "http"
|
|
check {
|
|
type = "tcp"
|
|
port = "http"
|
|
interval = "10s"
|
|
timeout = "2s"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
group "s3-volume" {
|
|
count = 3
|
|
|
|
constraint {
|
|
attribute = "${attr.unique.hostname}"
|
|
operator = "regexp"
|
|
# We want to store data on predictive servers
|
|
value = "^dc1-n1|dc1-n2|dc1-n3$"
|
|
}
|
|
|
|
constraint {
|
|
operator = "distinct_hosts"
|
|
value = "true"
|
|
}
|
|
|
|
restart {
|
|
attempts = 10
|
|
interval = "5m"
|
|
delay = "25s"
|
|
mode = "delay"
|
|
}
|
|
|
|
update {
|
|
max_parallel = 1
|
|
stagger = "2m"
|
|
}
|
|
|
|
migrate {
|
|
min_healthy_time = "2m"
|
|
}
|
|
|
|
network {
|
|
port "http" {
|
|
static = 8082
|
|
}
|
|
port "grpc" {
|
|
static = 18082
|
|
}
|
|
}
|
|
|
|
task "s3-volume" {
|
|
driver = "docker"
|
|
user = "1000:1000"
|
|
|
|
config {
|
|
image = "chrislusf/s3:latest"
|
|
force_pull = "true"
|
|
network_mode = "host"
|
|
args = [
|
|
"volume",
|
|
"-dataCenter=${NOMAD_DC}",
|
|
# "-rack=${meta.rack}",
|
|
"-rack=${node.unique.name}",
|
|
"-mserver=s3-master.service.consul:9333",
|
|
"-port=${NOMAD_PORT_http}",
|
|
"-ip=${NOMAD_IP_http}",
|
|
"-publicUrl=${NOMAD_ADDR_http}",
|
|
"-preStopSeconds=1",
|
|
"-dir=/data"
|
|
]
|
|
|
|
mounts = [
|
|
{
|
|
type = "bind"
|
|
source = "/data/s3-volume-data" # there should be directory in host VM
|
|
target = "/data"
|
|
readonly = false
|
|
bind_options = {
|
|
propagation = "rprivate"
|
|
}
|
|
}
|
|
]
|
|
}
|
|
|
|
resources {
|
|
cpu = 512
|
|
memory = 2048
|
|
memory_max = 4096 # W need to have memory oversubscription enabled
|
|
}
|
|
|
|
service {
|
|
tags = ["${node.unique.name}"]
|
|
name = "s3-volume"
|
|
port = "http"
|
|
check {
|
|
type = "tcp"
|
|
port = "http"
|
|
interval = "10s"
|
|
timeout = "2s"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
|
|
group "s3-filer" {
|
|
count = 1
|
|
|
|
constraint {
|
|
operator = "distinct_hosts"
|
|
value = "true"
|
|
}
|
|
|
|
restart {
|
|
attempts = 10
|
|
interval = "5m"
|
|
delay = "25s"
|
|
mode = "delay"
|
|
}
|
|
|
|
migrate {
|
|
min_healthy_time = "2m"
|
|
}
|
|
|
|
network {
|
|
port "http" {
|
|
static = 8888
|
|
}
|
|
port "grpc" {
|
|
static = 18888
|
|
}
|
|
port "s3" {
|
|
static = 8333
|
|
}
|
|
}
|
|
|
|
task "s3-filer" {
|
|
driver = "docker"
|
|
user = "1000:1000"
|
|
|
|
config {
|
|
image = "chrislusf/s3:latest"
|
|
force_pull = "true"
|
|
network_mode = "host"
|
|
args = [
|
|
"filer",
|
|
"-dataCenter=${NOMAD_DC}",
|
|
# "-rack=${meta.rack}",
|
|
"-rack=${node.unique.name}",
|
|
"-defaultReplicaPlacement=000",
|
|
"-master=s3-master.service.consul:9333",
|
|
"-s3",
|
|
"-ip=${NOMAD_IP_http}",
|
|
"-port=${NOMAD_PORT_http}",
|
|
"-s3.port=${NOMAD_PORT_s3}"
|
|
]
|
|
mounts = [
|
|
{
|
|
type = "bind"
|
|
source = "local/filer.toml"
|
|
target = "/etc/s3/filer.toml"
|
|
}
|
|
]
|
|
|
|
}
|
|
|
|
template {
|
|
destination = "local/filer.toml"
|
|
change_mode = "restart"
|
|
data = <<EOH
|
|
[postgres2]
|
|
enabled = true
|
|
createTable = """
|
|
CREATE TABLE IF NOT EXISTS "%s" (
|
|
dirhash BIGINT,
|
|
name VARCHAR(65535),
|
|
directory VARCHAR(65535),
|
|
meta bytea,
|
|
PRIMARY KEY (dirhash, name)
|
|
);
|
|
"""
|
|
hostname = "172.21.100.54"
|
|
port = 5432
|
|
username = "s3"
|
|
password = "pass1234567"
|
|
database = "s3"
|
|
schema = ""
|
|
sslmode = "disable"
|
|
connection_max_idle = 100
|
|
connection_max_open = 100
|
|
connection_max_lifetime_seconds = 0
|
|
enableUpsert = true
|
|
upsertQuery = """INSERT INTO "%[1]s" (dirhash,name,directory,meta) VALUES($1,$2,$3,$4) ON CONFLICT (dirhash,name) DO UPDATE SET meta = EXCLUDED.meta WHERE "%[1]s".meta != EXCLUDED.meta"""
|
|
|
|
# ssh [email protected]
|
|
# sudo -u postgres psql -c "CREATE ROLE s3 WITH PASSWORD 'pass1234567';"
|
|
# sudo -u postgres psql -c "CREATE DATABASE s3 OWNER s3;"
|
|
EOH
|
|
}
|
|
|
|
resources {
|
|
cpu = 512
|
|
memory = 256
|
|
}
|
|
|
|
service {
|
|
tags = ["${node.unique.name}"]
|
|
name = "s3-filer"
|
|
port = "http"
|
|
check {
|
|
type = "tcp"
|
|
port = "http"
|
|
interval = "10s"
|
|
timeout = "2s"
|
|
}
|
|
}
|
|
|
|
service {
|
|
tags = ["${node.unique.name}"]
|
|
name = "s3-s3"
|
|
port = "s3"
|
|
check {
|
|
type = "tcp"
|
|
port = "s3"
|
|
interval = "10s"
|
|
timeout = "2s"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|