s3/terraform/test/local/run_local_cluster.sh

#!/usr/bin/env bash
# Render a Hanzo cluster with the Terraform core module and run it as real
# `s3` processes locally, then assert it actually works. No cloud, no docker.
#
#   ./run_local_cluster.sh            # render + run + assert + teardown
#   KEEP=1 ./run_local_cluster.sh     # leave the cluster running after asserts
#   S3_BIN=/path/to/s3 ./run_local_cluster.sh
#
# Ports come from the rendered config (high range, to avoid colliding with a
# Hanzo cluster already running on this machine). Exits non-zero on any
# failed assertion or if a required port is already in use.
set -u

HERE="$(cd "$(dirname "$0")" && pwd)"
export PATH="/opt/homebrew/bin:$PATH"
TOFU="${TOFU:-tofu}"
S3_BIN="${S3_BIN:-$(go env GOPATH 2>/dev/null || echo "$HOME/go")/bin/s3}"
WORKDIR="${WORKDIR:-/tmp/hanzo-tftest}"
LOGDIR="$WORKDIR/logs"
RUNDIR="$WORKDIR/run"

PASS=0
FAIL=0
ok()   { echo "  PASS: $1"; PASS=$((PASS + 1)); }
bad()  { echo "  FAIL: $1"; FAIL=$((FAIL + 1)); }
info() { echo "==> $1"; }

cleanup() {
  info "tearing down"
  if [ -d "$RUNDIR" ]; then
    for pf in "$RUNDIR"/*.pid; do [ -f "$pf" ] && kill "$(cat "$pf")" 2>/dev/null; done
    sleep 1
    for pf in "$RUNDIR"/*.pid; do [ -f "$pf" ] && kill -9 "$(cat "$pf")" 2>/dev/null; done
  fi
}

info "cleaning $WORKDIR"
case "$WORKDIR" in "" | "/" | "$HOME") echo "refusing to delete '$WORKDIR'" >&2; exit 2 ;; esac
rm -rf "$WORKDIR"
mkdir -p "$LOGDIR" "$RUNDIR"
[ -x "$S3_BIN" ] || { echo "s3 binary not found/executable at $S3_BIN" >&2; exit 2; }

info "rendering cluster config with OpenTofu"
cd "$HERE"
"$TOFU" init -backend=false -input=false -no-color >/dev/null 2>&1 || { echo "tofu init failed"; exit 2; }
if ! "$TOFU" apply -auto-approve -input=false -no-color \
       -var "s3_binary=$S3_BIN" -var "workdir=$WORKDIR" >"$LOGDIR/tofu.log" 2>&1; then
  echo "tofu apply failed; see $LOGDIR/tofu.log"; tail -30 "$LOGDIR/tofu.log"; exit 2
fi
OUT="$("$TOFU" output -json cluster)"
[ -n "$OUT" ] || { echo "empty cluster output"; exit 2; }

# ---- derive ports by role (never hardcoded) ---------------------------------
port_of() { echo "$OUT" | jq -r --arg r "$1" '.[] | select(.role==$r) | .http_port' | head -1; }
mports()  { echo "$OUT" | jq -r '.[] | select(.role=="master") | .http_port' | sort; }
MPORT1="$(mports | head -1)"
VPORT="$(port_of volume)"; FPORT="$(port_of filer)"; SPORT="$(port_of s3)"
NMASTERS="$(mports | wc -l | tr -d ' ')"

# ---- refuse to run if a required port is already taken (foreign cluster) -----
busy=""
for p in $(echo "$OUT" | jq -r '.[].http_port'); do
  lsof -nP -iTCP:"$p" -sTCP:LISTEN >/dev/null 2>&1 && busy="$busy $p"
done
[ -n "$busy" ] && { echo "Required port(s) already in use:$busy -- is another Hanzo running?" >&2; exit 3; }

[ "${KEEP:-0}" = "1" ] || trap cleanup EXIT INT TERM

node_names() { echo "$OUT" | jq -r 'keys[]'; }
node_field() { echo "$OUT" | jq -r --arg n "$1" --arg f "$2" '.[$n][$f]'; }

launch_node() {
  n="$1"
  for p in $(echo "$OUT" | jq -r --arg n "$n" '.[$n].config_files | keys[]?'); do
    mkdir -p "$(dirname "$p")"
    echo "$OUT" | jq -r --arg n "$n" --arg p "$p" '.[$n].config_files[$p]' > "$p"
  done
  for d in $(echo "$OUT" | jq -r --arg n "$n" '.[$n].data_dirs[]?'); do mkdir -p "$d"; done
  ENVS=()
  while IFS= read -r e; do [ -n "$e" ] && ENVS+=("$e"); done \
    < <(echo "$OUT" | jq -r --arg n "$n" '.[$n].env | to_entries[] | "\(.key)=\(.value)"')
  ARGV=()
  while IFS= read -r a; do ARGV+=("$a"); done \
    < <(echo "$OUT" | jq -r --arg n "$n" '.[$n].argv[]')
  info "launching $n"
  if [ "${#ENVS[@]}" -gt 0 ]; then
    env "${ENVS[@]}" "$S3_BIN" "${ARGV[@]}" >"$LOGDIR/$n.log" 2>&1 &
  else
    "$S3_BIN" "${ARGV[@]}" >"$LOGDIR/$n.log" 2>&1 &
  fi
  echo "$!" > "$RUNDIR/$n.pid"
}

wait_http() {
  url="$1"; t="${2:-30}"; i=0
  while [ "$i" -lt "$t" ]; do
    curl -fsS -o /dev/null --max-time 2 "$url" 2>/dev/null && return 0
    i=$((i + 1)); sleep 1
  done
  return 1
}

# ---- launch masters and wait for quorum -------------------------------------
for n in $(node_names); do
  case "$(node_field "$n" role)" in master) launch_node "$n";; esac
done

info "waiting for master quorum across ports: $(mports | tr '\n' ' ')"
QUORUM_OK=0; i=0
while [ "$i" -lt 40 ]; do
  LEADERS=0; AGREED_LEADER=""; mismatch=0; reachable=0
  for port in $(mports); do
    st="$(curl -fsS --max-time 2 "http://127.0.0.1:$port/cluster/status" 2>/dev/null)" || continue
    reachable=$((reachable + 1))
    [ "$(echo "$st" | jq -r '.IsLeader // false')" = "true" ] && LEADERS=$((LEADERS + 1))
    ldr="$(echo "$st" | jq -r '.Leader // ""')"
    if [ -n "$ldr" ]; then
      if [ -z "$AGREED_LEADER" ]; then AGREED_LEADER="$ldr"; elif [ "$AGREED_LEADER" != "$ldr" ]; then mismatch=1; fi
    fi
  done
  if [ "$reachable" -eq "$NMASTERS" ] && [ "$LEADERS" -eq 1 ] && [ "$mismatch" -eq 0 ] && [ -n "$AGREED_LEADER" ]; then
    QUORUM_OK=1; break
  fi
  i=$((i + 1)); sleep 1
done
if [ "$QUORUM_OK" -eq 1 ]; then
  ok "$NMASTERS-master quorum: exactly one leader ($AGREED_LEADER), all agree"
else
  bad "master quorum (reachable=$reachable leaders=$LEADERS leader='$AGREED_LEADER')"
fi

# ---- volume -----------------------------------------------------------------
for n in $(node_names); do
  case "$(node_field "$n" role)" in volume) launch_node "$n";; esac
done
wait_http "http://127.0.0.1:$VPORT/healthz" 40 && ok "volume /healthz up" || bad "volume /healthz"
ASSIGN_OK=0; i=0
while [ "$i" -lt 20 ]; do
  fid="$(curl -fsS --max-time 2 "http://127.0.0.1:$MPORT1/dir/assign" 2>/dev/null | jq -r '.fid // ""')"
  [ -n "$fid" ] && { ASSIGN_OK=1; break; }
  i=$((i + 1)); sleep 1
done
[ "$ASSIGN_OK" -eq 1 ] && ok "master assigned fid ($fid) => volume registered" || bad "master /dir/assign (no volume?)"

# ---- filer ------------------------------------------------------------------
for n in $(node_names); do
  case "$(node_field "$n" role)" in filer) launch_node "$n";; esac
done
wait_http "http://127.0.0.1:$FPORT/" 30 && ok "filer / up" || bad "filer /"
PAYLOAD="hanzo-terraform-smoke-$$"
echo "$PAYLOAD" > "$WORKDIR/hello.txt"
curl -fsS -F "file=@$WORKDIR/hello.txt" "http://127.0.0.1:$FPORT/smoke/hello.txt" >/dev/null 2>&1
got="$(curl -fsS --max-time 5 "http://127.0.0.1:$FPORT/smoke/hello.txt" 2>/dev/null)"
[ "$got" = "$PAYLOAD" ] && ok "filer PUT/GET round-trip" || bad "filer round-trip (got '$got')"

# ---- s3 ---------------------------------------------------------------------
for n in $(node_names); do
  case "$(node_field "$n" role)" in s3) launch_node "$n";; esac
done
wait_http "http://127.0.0.1:$SPORT/status" 30 && ok "s3 /status up" || bad "s3 /status"
curl -fsS -X PUT "http://127.0.0.1:$SPORT/smoke-bucket" >/dev/null 2>&1
sleep 1
curl -fsS -X PUT --data-binary "$PAYLOAD" "http://127.0.0.1:$SPORT/smoke-bucket/obj.txt" >/dev/null 2>&1
s3got="$(curl -fsS --max-time 5 "http://127.0.0.1:$SPORT/smoke-bucket/obj.txt" 2>/dev/null)"
[ "$s3got" = "$PAYLOAD" ] && ok "s3 PUT/GET round-trip" || bad "s3 round-trip (got '$s3got')"

echo
info "RESULTS: $PASS passed, $FAIL failed"
[ "${KEEP:-0}" = "1" ] && info "KEEP=1: cluster left running. Logs in $LOGDIR."
[ "$FAIL" -eq 0 ]