universe/deploy/pin.sh

#!/usr/bin/env bash
#
# pin — move a service's image, and be the only thing that does.
#
# cd.hanzo.ai does not watch the registry. It watches this directory: the
# lux-fleet ApplicationSet (applicationset.yaml) has a git generator over
# deploy/*/*.yaml, and each file it finds becomes one Application whose values
# are that file. So publishing an image changes nothing. Moving the pin here is
# the deploy, and it is the whole deploy.
#
# A pin is TWO fields and ONE fact. `tag` is how a human says which release;
# `digest` is what the kubelet pulls. _helpers.tpl renders `repo:tag@digest`,
# and a reference carrying both RESOLVES BY DIGEST. Written from two registry
# reads they can name two different releases, and the pod then reports the tag
# while running the digest — a deploy that succeeds at doing nothing. So there
# is one read: `HEAD /v2/<path>/manifests/<tag>` answers the tag that was asked
# for, and Docker-Content-Digest is the digest OF THAT ANSWER. Both are written
# in one edit and read back together.
#
# Why pin the digest when the tag reads better: `pullPolicy: IfNotPresent` is
# what the fleet runs, so a node that ever pulled `:v1.2.3` keeps those bytes.
# Registry tags are mutable. The digest is the identity; the tag is the name.
#
#   deploy/pin.sh <namespace>/<service> <version>
#   deploy/pin.sh --verify [<namespace>/<service>…]
#   deploy/pin.sh --which <repository>          # who runs this image
#   deploy/pin.sh --freeze <namespace>/<service>…  # a moving tag names bytes
#
# Refusals, each a way a release can put something unintended into production:
#   non-semver — a sha cannot be read, ordered, or rolled back to by a human;
#   an unpublished tag — resolved BEFORE the pin moves, so a pin can never name
#     an image the registry does not have;
#   a caller-chosen repository — read from the values file, never from argv, so
#     a careless build can only move its own service between its own releases;
#   going backward — refused unless PIN_ROLLBACK=1, which is a different act;
#   a service with no values file — adding a service is deliberate.
#
# The invariants are charts/app/pin.sh's, which owns hanzo's fleet under
# charts/app/values/. That script's values root is fixed at its own directory,
# so it cannot be pointed at this one; when it takes a root, this file goes.

set -euo pipefail

cd "$(dirname "${BASH_SOURCE[0]}")"          # deploy/

die() { echo "::error::pin: $*" >&2; exit 1; }

# Being overtaken is not a failure. A release asking for a version the pin is
# already past has got what it wanted, so it gets a status of its own.
overtaken() { echo "::notice::pin: $*" >&2; exit 3; }

# Every manifest type a pullable image can be stored as. A registry answers 404
# when the tag exists but is stored as a type the request does not accept, which
# is indistinguishable here from a tag that was never published.
ACCEPT='application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json'

# A bearer token out of a registry's token endpoint, or nothing. Prints nothing
# rather than a traceback when the body is not the JSON that was expected,
# because the caller's next line already says what a missing token means. GitLab
# and a few others answer `access_token` where the spec says `token`.
read_token() {
  python3 -c 'import sys,json
try:
    d = json.load(sys.stdin); sys.stdout.write(d.get("token") or d.get("access_token") or "")
except Exception: pass' 2>/dev/null
}

# THE ONE READ. Prints the digest, or `!<reason>` with exit 1 so the verifier can
# report every service instead of stopping at the first.
#
# AUTH IS ASKED FOR, NOT GUESSED. Every registry here answers a 401 that SAYS
# where its tokens come from — `Www-Authenticate: Bearer realm=…,service=…` — so
# the probe is sent unauthenticated first and the challenge is read off the
# refusal. Before this the realm was assembled per host from a table, which had
# ghcr's shape and therefore reported quay.io's three cert-manager images and
# registry.digitalocean.com's lux-admin as UNPULLABLE — four false alarms, from
# four registries that were serving normally. A table of hosts is a guess about
# other people's servers; the challenge is their own answer.
resolve_digest() {
  local repository="$1" tag="$2" first host path chal realm service token headers code digest
  first="${repository%%/*}"

  # Which registry a reference names is a property of the reference, and the rule
  # is the one every client uses: a first segment carrying a dot or a port is a
  # host; anything else is Docker Hub, where a bare name means library/.
  # Third-party images are resolved here too — openresty:alpine and
  # python:3.12-alpine are moving pointers exactly like :latest, and a base image
  # that changes under a service is harder to see than one that changes with it.
  if printf '%s' "$first" | grep -q '[.:]'; then
    host="$first"; path="${repository#*/}"
  else
    host="registry-1.docker.io"
    case "$repository" in */*) path="$repository" ;; *) path="library/$repository" ;; esac
  fi

  # HEAD, not GET: the digest is a header, so the body is bytes nobody reads.
  # -L because registry.k8s.io answers 307 to a regional mirror, and an
  # unfollowed redirect is indistinguishable here from a tag that is not there.
  probe() {
    curl -sSIL --max-time 30 -H "Accept: $ACCEPT" ${1:+-H "Authorization: Bearer $1"} \
      "https://${host}/v2/${path}/manifests/${tag}" 2>/dev/null
  }

  headers="$(probe "")" \
    || { echo "!manifest request for ${repository}:${tag} failed (network)"; return 1; }

  if printf '%s' "$headers" | head -1 | grep -q ' 401'; then
    chal="$(printf '%s' "$headers" | tr -d '\r' | sed -n 's/^[Ww][Ww][Ww]-[Aa]uthenticate: *[Bb]earer *//p' | head -1)"
    realm="$(printf '%s' "$chal" | sed -n 's/.*realm="\([^"]*\)".*/\1/p')"
    service="$(printf '%s' "$chal" | sed -n 's/.*service="\([^"]*\)".*/\1/p')"
    [ -n "$realm" ] || { echo "!${repository}:${tag} refused anonymously and named no token realm"; return 1; }

    # Credentials are offered only where a private image needs them. They are in
    # scope in any build that pushed the image it is now pinning.
    if [ -n "${GHCR_USER:-}" ] && [ -n "${GHCR_TOKEN:-}" ] && [ "$host" = "ghcr.io" ]; then
      token="$(curl -fsSL --max-time 30 -u "$GHCR_USER:$GHCR_TOKEN" \
        "${realm}?scope=repository:${path}:pull&service=${service}" 2>/dev/null | read_token)" || token=""
    else
      token="$(curl -fsSL --max-time 30 \
        "${realm}?scope=repository:${path}:pull&service=${service}" 2>/dev/null | read_token)" || token=""
    fi
    [ -n "$token" ] || { echo "!could not obtain a pull token for ${host}/${path}"; return 1; }

    headers="$(probe "$token")" \
      || { echo "!manifest request for ${repository}:${tag} failed (network or auth)"; return 1; }
  fi

  # The LAST status line and the LAST digest header: -L leaves one set of headers
  # per hop, and only the final hop answered.
  code="$(printf '%s' "$headers" | grep -E '^HTTP/' | tail -1 | awk '{print $2}')"
  [ "$code" = "200" ] || { echo "!${repository}:${tag} is not pullable (manifest probe returned ${code:-none})"; return 1; }

  digest="$(printf '%s' "$headers" | tr -d '\r' | sed -n 's/^[Dd]ocker-[Cc]ontent-[Dd]igest: *//p' | tail -1)"
  # A 200 with no header would let the caller write `repo:tag@`, which nothing
  # can pull. Refuse instead.
  [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] \
    || { echo "!${repository}:${tag} answered 200 but named no Docker-Content-Digest"; return 1; }
  echo "$digest"
}

# Read one key out of a values file's TOP-LEVEL image: block. Indented comments
# stay in the block — these files carry the reasoning for every value in them —
# so only a line starting in column 0 ends it. A tag YAML would read as a number
# is written quoted; the quotes are syntax, not part of the name.
image_field() {
  awk -v key="$2" '
    /^image:/                   { in_image = 1; next }
    in_image && /^[^[:space:]]/ { in_image = 0 }
    in_image && $1 == key":"    { v = $2; gsub(/^[\047"]|[\047"]$/, "", v); print v; exit }
  ' "$1"
}

# ONE EDIT, in place. These files carry the reasoning for every value in them, so
# a YAML round-trip would reflow it away. A file that never carried a digest gains
# one directly beneath the tag, at its indentation — the pair is one fact.
#
# Both scalars are written by the same pass and read back together, because the
# edit is the only place they can come apart. A fourth argument rewrites
# pullPolicy at the same time: freezing a moving tag and stopping the re-pull are
# one decision, not two, and doing them in separate passes is how a file ends up
# with a digest it re-resolves on every restart.
write_pin() {
  local values="$1" tag="$2" dig="$3" policy="${4:-}" has=0 q="" wrote_tag wrote_dig
  [ -n "$(image_field "$values" digest)" ] && has=1

  # A TAG IS A STRING, AND SOME STRINGS ARE NUMBERS. `1.17` written bare is a
  # YAML float, and a float loses its trailing zero: `1.10` reads back as 1.1 and
  # the chart then asks for an image the registry does not have. So a tag YAML
  # would read as a number is written quoted. This replaced the whole token,
  # quotes included, and consul's `tag: '1.17'` came back bare on the first run.
  [[ "$tag" =~ ^[+-]?[0-9]+([.][0-9]*)?$ ]] && q="'"

  awk -v tag="$tag" -v dig="$dig" -v has_digest="$has" -v policy="$policy" -v q="$q" '
    /^image:/                   { in_image = 1; print; next }
    in_image && /^[^[:space:]]/ { in_image = 0 }
    in_image && $1 == "tag:" && !wrote_tag {
        match($0, /^[[:space:]]*/); ind = substr($0, 1, RLENGTH)
        sub(/tag:[[:space:]]*[^[:space:]#]+/, "tag: " q tag q)
        print; wrote_tag = 1
        if (has_digest == 0) print ind "digest: " dig
        next }
    in_image && $1 == "digest:" && !wrote_digest {
        sub(/digest:[[:space:]]*[^[:space:]#]*/, "digest: " dig)
        print; wrote_digest = 1; next }
    in_image && $1 == "pullPolicy:" && policy != "" {
        sub(/pullPolicy:[[:space:]]*[^[:space:]#]+/, "pullPolicy: " policy)
        print; next }
    { print }
  ' "$values" > "$values.pin" && mv "$values.pin" "$values"

  wrote_tag="$(image_field "$values" tag)"
  wrote_dig="$(image_field "$values" digest)"
  [ "$wrote_tag" = "$tag" ] \
    || die "wrote deploy/$values but image.tag reads back as '$wrote_tag', expected '$tag'"
  [ "$wrote_dig" = "$dig" ] \
    || die "wrote deploy/$values but image.digest reads back as '$wrote_dig', expected '$dig' — the file names a tag and a digest that disagree, which deploys the digest"
}

# A service resolves to exactly one values file. Zero means it is not in the
# inventory; more than one means two namespaces hold the name and the caller
# must say which — the same collision the ApplicationSet's name template guards.
resolve_values() {
  local pat line; local -a m=()
  case "$1" in
    */*) pat="$1.yaml" ;;
    *)   pat="./*/$1.yaml" ;;
  esac
  while IFS= read -r line; do [ -n "$line" ] && m+=("$line"); done < <(ls -1 $pat 2>/dev/null)
  case "${#m[@]}" in
    0) die "no values file for '$1' — a service is added deliberately, not by a release" ;;
    1) echo "${m[0]#./}" ;;
    *) die "'$1' is declared in ${#m[@]} namespaces (${m[*]}) — name one as <namespace>/$1" ;;
  esac
}

# ── VERIFY ────────────────────────────────────────────────────────────────────
#
# Every values file naming a digest must name the digest its own tag resolves to.
# Run over the whole inventory in CI: the failure this catches is silent, and the
# only moment it is cheap to catch is before it merges.
verify() {
  local -a files=(); local s line
  if [ "$#" -gt 0 ]; then
    for s in "$@"; do files+=("$(resolve_values "$s")"); done
  else
    while IFS= read -r line; do files+=("$line"); done < <(ls -1 ./*/*.yaml | sed 's|^\./||')
  fi

  local bad=0 checked=0 f repository tag digest live name
  for f in "${files[@]}"; do
    repository="$(image_field "$f" repository)"
    tag="$(image_field "$f" tag)"
    digest="$(image_field "$f" digest)"
    name="${f%.yaml}"

    [ -n "$repository" ] && [ -n "$tag" ] || { printf '  %-44s %s\n' "$name" "— no tag to check"; continue; }
    checked=$((checked + 1))

    live="$(resolve_digest "$repository" "$tag")" || {
      printf '  %-44s %-26s %s\n' "$name" "$tag" "UNPULLABLE ${live#!}"; bad=$((bad + 1)); continue; }

    if [ -z "$digest" ]; then
      printf '  %-44s %-26s %s\n' "$name" "$tag" "tag only (resolves ${live:0:19}…)"
    elif [ "$digest" = "$live" ]; then
      printf '  %-44s %-26s %s\n' "$name" "$tag" "ok ${live:0:19}…"
    else
      printf '  %-44s %-26s %s\n' "$name" "$tag" "DRIFT"
      printf '  %-44s   pinned digest: %s\n' "" "$digest"
      printf '  %-44s   tag resolves : %s\n' "" "$live"
      bad=$((bad + 1))
    fi
  done

  echo
  [ "$bad" -eq 0 ] || { echo "::error::pin --verify: $bad of $checked pins do not agree with the registry"; return 1; }
  echo "pin --verify: $checked pins agree with the registry"
}

# ── WHICH ─────────────────────────────────────────────────────────────────────
#
# WHO RUNS THIS IMAGE. A release knows the repository it published; it does not
# know, and must not have to be told, which namespaces run it. The directory IS
# the inventory, so the answer is read from it rather than kept in a second list
# that has to be edited when a service moves — the list this replaces named four
# paths that did not exist.
#
# Prints `<namespace>/<service>` per line, or nothing, which is not an error: an
# image with no pin here is published and simply not deployed by this fleet.
which_services() {
  local repository="$1" f
  [ -n "$repository" ] || die "usage: pin.sh --which <repository>"
  for f in ./*/*.yaml; do
    [ "$(image_field "$f" repository)" = "$repository" ] || continue
    f="${f#./}"; echo "${f%.yaml}"
  done
}

# ── FREEZE ────────────────────────────────────────────────────────────────────
#
# WHICH BYTES DOES THIS MOVING NAME MEAN TODAY.
#
# `latest`, `main` and `alpine` are pointers, not versions. Two nodes pulling the
# same values file a week apart run different code and both report the same tag,
# and there is nothing to roll back to because the name that was serving now
# means something else. Freeze records the digest the tag resolves to right now
# and leaves the tag beside it, so the reference reads `repo:latest@sha256:…` —
# still legible, no longer ambiguous.
#
# pullPolicy goes to IfNotPresent in the same edit. A digest IS an identity;
# re-pulling it every restart asks the registry a question whose answer cannot
# have changed, and leaving Always beside a digest reads as if it could.
#
# This does not replace a release. It stops a name from drifting until the
# service publishes versions and pin.sh <service> <version> can move it.
freeze() {
  local -a files=(); local s line
  if [ "$#" -gt 0 ]; then
    for s in "$@"; do files+=("$(resolve_values "$s")"); done
  else
    die "usage: pin.sh --freeze <namespace>/<service>…"
  fi

  local f repository tag digest live
  for f in "${files[@]}"; do
    repository="$(image_field "$f" repository)"
    tag="$(image_field "$f" tag)"
    digest="$(image_field "$f" digest)"
    [ -n "$repository" ] && [ -n "$tag" ] || { echo "  ${f%.yaml}: no tag to freeze"; continue; }
    live="$(resolve_digest "$repository" "$tag")" \
      || die "${live#!} — refusing to freeze a tag the registry does not have"
    [ "$digest" = "$live" ] && { echo "  ${f%.yaml}: already frozen at $live"; continue; }
    write_pin "$f" "$tag" "$live" IfNotPresent
    echo "  ${f%.yaml}: ${repository}:${tag} -> $live"
  done
}

if [ "${1:-}" = "--verify" ]; then shift; verify "$@"; exit $?; fi
if [ "${1:-}" = "--which" ];  then shift; which_services "${1:-}"; exit 0; fi
if [ "${1:-}" = "--freeze" ]; then shift; freeze "$@"; exit $?; fi

# ── PIN ───────────────────────────────────────────────────────────────────────

SERVICE="${1:-}"; VERSION="${2:-}"
[ -n "$SERVICE" ] && [ -n "$VERSION" ] \
  || die "usage: pin.sh <namespace>/<service> <version> | pin.sh --verify [<namespace>/<service>…]"

# Accept a leading v so a caller can pass its git tag or its image tag. The bare
# number is compared; the PREFIX comes from the values file, because the fleet is
# not consistent about it and pinning one convention writes tags that do not exist.
VERSION="${VERSION#v}"
[[ "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] \
  || die "version '$VERSION' is not semver x.y.z — refusing to pin (semver only, never a sha)"

VALUES="$(resolve_values "$SERVICE")"
REPOSITORY="$(image_field "$VALUES" repository)"
CURRENT="$(image_field "$VALUES" tag)"
CURRENT_DIGEST="$(image_field "$VALUES" digest)"
[ -n "$REPOSITORY" ] || die "deploy/$VALUES declares no image.repository"
[ -n "$CURRENT" ]    || die "deploy/$VALUES declares no image.tag"

CURRENT_BARE="${CURRENT#v}"

# Only a VERSION carries an order or a convention, and the current tag is not
# guaranteed to be one — a floating `latest` or a hand-written sha is exactly
# when the pin still has to work. So nothing below is derived from a value
# already known to be unreadable: a sha and a semver compare as strings, and
# `sort -V` would read a forward release as a rollback.
if [[ "$CURRENT_BARE" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
  PREFIX=""; case "$CURRENT" in v*) PREFIX="v" ;; esac
  TAG="${PREFIX}${VERSION}"

  # Checked before the same-version short-circuit: a rollback and a no-op are
  # different acts and only one of them may be silent.
  if [ "$CURRENT_BARE" != "$VERSION" ] && [ "${PIN_ROLLBACK:-0}" != "1" ] \
     && [ "$(printf '%s\n%s\n' "$CURRENT_BARE" "$VERSION" | sort -V | head -1)" = "$VERSION" ]; then
    overtaken "$SERVICE is pinned at $CURRENT and $VERSION is older — leaving it (PIN_ROLLBACK=1 to go back deliberately)"
  fi

  DIGEST="$(resolve_digest "$REPOSITORY" "$TAG")" \
    || die "${DIGEST#!} — refusing to pin an image the registry does not have"
else
  # The file names no convention, so the registry names it. Exactly one spelling
  # may answer: two make the tag ambiguous, zero means it was never published,
  # and either turns the recovery into a guess.
  V_DIGEST="$(resolve_digest "$REPOSITORY" "v$VERSION")" || V_DIGEST=""
  B_DIGEST="$(resolve_digest "$REPOSITORY" "$VERSION")"  || B_DIGEST=""
  if [ -n "$V_DIGEST" ] && [ -n "$B_DIGEST" ]; then
    die "$REPOSITORY publishes BOTH v$VERSION and $VERSION, and deploy/$VALUES names no convention to choose by (its tag '$CURRENT' is not a version) — set image.tag to the intended spelling and re-run"
  elif [ -n "$V_DIGEST" ]; then TAG="v$VERSION"; DIGEST="$V_DIGEST"
  elif [ -n "$B_DIGEST" ]; then TAG="$VERSION";  DIGEST="$B_DIGEST"
  else die "neither v$VERSION nor $VERSION is pullable from $REPOSITORY — refusing to pin an image the registry does not have"
  fi
  echo "pin: $SERVICE carried '$CURRENT', which is not a version — the registry names the convention '$TAG'"
fi

# The only early exit, and only when BOTH halves already name what the registry
# just answered. A file whose tag is current but whose digest is stale is the bug
# this exists to end, so it falls through and is corrected.
if [ "$CURRENT_BARE" = "$VERSION" ] && [ "$CURRENT_DIGEST" = "$DIGEST" ]; then
  echo "pin: $SERVICE already at $CURRENT ($DIGEST) — nothing to do"; exit 0
fi

write_pin "$VALUES" "$TAG" "$DIGEST"

REPO_ROOT="$(git rev-parse --show-toplevel)"
git -C "$REPO_ROOT" add "deploy/$VALUES"
if git -C "$REPO_ROOT" diff --cached --quiet; then
  echo "pin: deploy/$VALUES already reads $TAG@$DIGEST at HEAD — nothing to commit"; exit 0
fi

# WHO MOVED THIS PIN. The author is the owner and Hanzo Dev is the co-author,
# whoever runs it, so the author cannot say "did CI deploy, or did a person?" —
# the question that says whether the automation works. CI sets GITHUB_RUN_ID;
# a person at a terminal does not, and Pinned-by records which it was.
PINNED_BY="hand"
[ -n "${GITHUB_RUN_ID:-}" ] && PINNED_BY="ci run ${GITHUB_RUN_ID}"

git -C "$REPO_ROOT" -c user.name="Zach Kelling" -c user.email="[email protected]" \
  commit --trailer "Co-authored-by: Hanzo Dev <[email protected]>" -m "$SERVICE $CURRENT -> $TAG

${REPOSITORY}:${TAG}
${DIGEST}

Tag and digest are one registry answer, resolved together and written in one
edit, so the name a human reads and the bytes the kubelet pulls cannot be two
different releases. cd.hanzo.ai reconciles the change from here.

Pinned-by: ${PINNED_BY}"

echo "pin: $SERVICE $CURRENT -> $TAG ($DIGEST) [deploy/$VALUES]"