Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 hanzo-dev
|
f098b39ecd | ||
|
hanzo-dev
|
674fabdcba | ||
|
hanzo-dev
|
3f65c9e24f | ||
 zeekayandhanzo-dev
|
8e43277aab | ||
|
zeekayandClaude Fable 5
|
d3823df85a | ||
|
zeekay
|
b6b52b4a81 | ||
|
zeekay
|
331e05625f | ||
|
zeekay
|
44186bea06 |
@@ -0,0 +1,60 @@
|
||||
name: sitedeploy
|
||||
description: Publish a built static export to the Hanzo PaaS Sites plane.
|
||||
# For every repo whose deploy is "a static export goes live" — hanzo.ai, hanzo.app,
|
||||
# hips, computer and the ones after them. Those were each about to carry their own
|
||||
# copy of enqueue → upload → complete, which is how one contract becomes N
|
||||
# transcriptions that drift (see the imgver action next door for the same story
|
||||
# told about image tags).
|
||||
#
|
||||
# - uses: hanzoai/ci/.github/actions/sitedeploy@v1
|
||||
# with: { slug: hanzo-ai, dir: out }
|
||||
# env: { HANZO_DEPLOY_TOKEN: '${{ secrets.HANZO_DEPLOY_TOKEN }}' }
|
||||
#
|
||||
# A SITE IS NOT AN APP. This publishes files and stops: no image, no CR, no
|
||||
# replicas, no registry. Building a container so a Go binary can serve /public is
|
||||
# the shape the Sites plane exists to retire.
|
||||
#
|
||||
# ONE credential. The 202 hands back a prefix-scoped, 30-minute presigned POST
|
||||
# grant, so CI never holds a bucket key — do NOT add SITES_S3_* here; that is the
|
||||
# standing shared-bucket credential the grant replaced.
|
||||
inputs:
|
||||
slug:
|
||||
description: The project slug on the Sites plane (POST /v1/projects/<slug>/deploy)
|
||||
required: true
|
||||
dir:
|
||||
description: The built export directory
|
||||
required: true
|
||||
api:
|
||||
description: Cloud API base
|
||||
required: false
|
||||
default: https://api.hanzo.ai
|
||||
jobs:
|
||||
description: Parallel uploads
|
||||
required: false
|
||||
default: '24'
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Fetch sitedeploy
|
||||
shell: bash
|
||||
# The action ref is the script ref: an action pinned to @v1 runs v1's
|
||||
# sitedeploy. Both forges, because this repo is served from each.
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ref="${GITHUB_ACTION_REF:-v1}"
|
||||
for url in https://git.hanzo.ai/hanzoai/ci https://github.com/hanzoai/ci; do
|
||||
git clone -q --depth 1 --branch "$ref" "$url" "$RUNNER_TEMP/sitedeploy-ci" 2>/dev/null && break
|
||||
done
|
||||
[ -x "$RUNNER_TEMP/sitedeploy-ci/bin/sitedeploy" ] \
|
||||
|| { echo "::error::could not fetch hanzoai/ci@$ref (bin/sitedeploy)"; exit 1; }
|
||||
|
||||
- name: Deploy
|
||||
shell: bash
|
||||
env:
|
||||
HANZO_API: ${{ inputs.api }}
|
||||
SITEDEPLOY_JOBS: ${{ inputs.jobs }}
|
||||
SITEDEPLOY_COMMIT: ${{ github.sha }}
|
||||
SITEDEPLOY_BRANCH: ${{ github.ref_name }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
bash "$RUNNER_TEMP/sitedeploy-ci/bin/sitedeploy" '${{ inputs.slug }}' '${{ inputs.dir }}'
|
||||
+316
-213
@@ -73,7 +73,7 @@ on:
|
||||
A string sidesteps it on both planes with identical meaning, and the
|
||||
comparison is against 'false' rather than a truthiness test so that the
|
||||
expression callers already write — hanzoai/cloud's
|
||||
`tests: ${{ github.ref_type != 'tag' }}`, which renders the strings
|
||||
`tests:` set from `github.ref_type != 'tag'`, which renders the strings
|
||||
"true"/"false" — keeps working unchanged.
|
||||
type: string
|
||||
default: 'true'
|
||||
@@ -104,6 +104,14 @@ permissions:
|
||||
jobs:
|
||||
cicd:
|
||||
runs-on: ${{ fromJson(inputs.runner) }}
|
||||
env:
|
||||
# corepack fetches the pnpm/yarn TARBALL ITSELF from registry.npmjs.org,
|
||||
# and it does not read .npmrc to do it — so 317 repos that pin
|
||||
# `packageManager` reach out to npmjs on every install no matter how
|
||||
# carefully their registry is configured. This is the one variable that
|
||||
# closes that hole; without it "installs go through our registry" is true
|
||||
# of the dependencies and false of the package manager fetching them.
|
||||
COREPACK_NPM_REGISTRY: https://pkg.hanzo.ai
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
@@ -427,6 +435,18 @@ jobs:
|
||||
# ask KMS twice.
|
||||
SITE_DECL="$(yq -r '.site | type' hanzo.yml 2>/dev/null || echo '!!null')"
|
||||
if [ -n "$(yq -r '.bucket // ""' hanzo.yml 2>/dev/null || true)" ] || [ "$SITE_DECL" = '!!map' ]; then
|
||||
# The registry credential comes from KMS like every other secret —
|
||||
# the ONLY things that live in GitHub are KMS_CLIENT_ID/SECRET.
|
||||
# Named here so a repo with no kubeconfig (most of them) can still
|
||||
# reach registry.hanzo.ai; the kubeconfig read below stays as the
|
||||
# fallback for repos provisioned before these names existed.
|
||||
for k in REGISTRY_USER REGISTRY_PASSWORD; do
|
||||
v=$(get "$k")
|
||||
if [ -n "$v" ]; then
|
||||
[ "$k" = REGISTRY_PASSWORD ] && echo "::add-mask::$v"
|
||||
echo "$k=$v" >> "$GITHUB_ENV"
|
||||
fi
|
||||
done
|
||||
for k in S3_ADMIN_ACCESS_KEY S3_ADMIN_SECRET_KEY; do
|
||||
v=$(get "$k")
|
||||
if [ -n "$v" ]; then echo "::add-mask::$v"; echo "$k=$v" >> "$GITHUB_ENV"; fi
|
||||
@@ -521,14 +541,20 @@ jobs:
|
||||
echo "::notice::KMS GHCR token login failed — keeping the earlier login"
|
||||
fi
|
||||
|
||||
- name: Native registry credential (oci.hanzo.ai)
|
||||
- name: Native registry credential (registry.hanzo.ai)
|
||||
if: inputs.mode != 'delegate'
|
||||
env:
|
||||
KUBECONFIG: ${{ steps.kms.outputs.kubeconfig }}
|
||||
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
|
||||
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
# KMS first (exported to GITHUB_ENV by the step above), GitHub secret
|
||||
# second. `env:` here cannot see GITHUB_ENV from THIS step's own env
|
||||
# block, so the KMS values are read out of the environment in `run:`.
|
||||
GH_REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
|
||||
GH_REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
# REGISTRY_USER/PASSWORD may already be in the environment from KMS.
|
||||
REGISTRY_USER="${REGISTRY_USER:-${GH_REGISTRY_USER:-}}"
|
||||
REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-${GH_REGISTRY_PASSWORD:-}}"
|
||||
# Direct credential first (repo/org secret — works on private repos,
|
||||
# where the Free plan hides org secrets... including these; private
|
||||
# repos set them at REPO level). KMS-kubeconfig read is the fallback.
|
||||
@@ -537,14 +563,14 @@ jobs:
|
||||
# cred) must never fail the run — the image still pushes to GHCR, the
|
||||
# primary. Without this guard, bash -e aborts the step and SKIPS the
|
||||
# build entirely (a registry hiccup takes the whole lane red).
|
||||
if echo "$REGISTRY_PASSWORD" | docker login oci.hanzo.ai -u "$REGISTRY_USER" --password-stdin; then
|
||||
if echo "$REGISTRY_PASSWORD" | docker login registry.hanzo.ai -u "$REGISTRY_USER" --password-stdin; then
|
||||
echo "MIRROR_OK=1" >> "$GITHUB_ENV"
|
||||
else
|
||||
echo "::notice::registry.hanzo.ai login failed — mirror skipped (GHCR-only push)"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
[ -z "${KUBECONFIG:-}" ] && { echo "::warning::no registry credential and no kubeconfig — image will NOT reach oci.hanzo.ai (ghcr only)"; exit 0; }
|
||||
[ -z "${KUBECONFIG:-}" ] && { echo "::warning::no registry credential and no kubeconfig — image will NOT reach registry.hanzo.ai (ghcr only)"; exit 0; }
|
||||
# Bare runners ship no kubectl — same static provision the deploy
|
||||
# step uses.
|
||||
command -v kubectl >/dev/null 2>&1 || {
|
||||
@@ -555,218 +581,18 @@ jobs:
|
||||
}
|
||||
CFG=$(kubectl -n hanzo get secret registry-credentials -o jsonpath='{.data.\.dockerconfigjson}' 2>/dev/null | base64 -d || true)
|
||||
if [ -z "$CFG" ]; then
|
||||
echo "::warning::registry-credentials not readable from this kubeconfig — image will NOT reach oci.hanzo.ai (ghcr only)"; exit 0
|
||||
echo "::warning::registry-credentials not readable from this kubeconfig — image will NOT reach registry.hanzo.ai (ghcr only)"; exit 0
|
||||
fi
|
||||
USERPASS=$(echo "$CFG" | jq -r '.auths["registry.hanzo.ai"].auth // empty' | base64 -d)
|
||||
[ -z "$USERPASS" ] && { echo "::warning::no registry auth in dockerconfig — image will NOT reach oci.hanzo.ai (ghcr only)"; exit 0; }
|
||||
[ -z "$USERPASS" ] && { echo "::warning::no registry auth in dockerconfig — image will NOT reach registry.hanzo.ai (ghcr only)"; exit 0; }
|
||||
echo "::add-mask::${USERPASS#*:}"
|
||||
# Best-effort: login failure → skip mirror, never fail the run (see above).
|
||||
if echo "${USERPASS#*:}" | docker login oci.hanzo.ai -u "${USERPASS%%:*}" --password-stdin; then
|
||||
if echo "${USERPASS#*:}" | docker login registry.hanzo.ai -u "${USERPASS%%:*}" --password-stdin; then
|
||||
echo "MIRROR_OK=1" >> "$GITHUB_ENV"
|
||||
else
|
||||
echo "::notice::registry.hanzo.ai login failed — mirror skipped (GHCR-only push)"
|
||||
fi
|
||||
|
||||
- name: Build & push images (per hanzo.yml)
|
||||
if: inputs.mode != 'delegate'
|
||||
env:
|
||||
GH_PAT: ${{ secrets.GH_PAT }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Test-only callers (hanzo.yml without `images:` — e.g. a repo whose
|
||||
# image lane lives in its own release.yml, or a pure library) skip the
|
||||
# build step entirely instead of exploding on a null .images.
|
||||
if [ "$(yq -r '.images // [] | length' hanzo.yml 2>/dev/null || echo 0)" = "0" ]; then
|
||||
echo "::notice::no images: in hanzo.yml — test-only caller, skipping build"; exit 0
|
||||
fi
|
||||
# Build-time private cross-org Go module read (the buildx `gh_token`
|
||||
# secret): prefer the KMS-fetched GIT_TOKEN, else fall back to the org
|
||||
# GH_PAT — the SAME BuildKit gh_token cloud's release.yml uses (proven
|
||||
# working). Keeps image builds green when the KMS deploy-cred fetch is
|
||||
# unavailable (a repo with a private cross-org dep like hanzoai/cloud
|
||||
# otherwise fails `go mod tidy` with git exit 128 in the buildx stage).
|
||||
# No-op for public-only builds when both are empty. Exported so the
|
||||
# `--secret id=gh_token,env=GIT_TOKEN` below reads it from the env.
|
||||
export GIT_TOKEN="${GIT_TOKEN:-${GH_PAT:-}}"
|
||||
SHORT=$(echo "${{ github.sha }}" | cut -c1-7)
|
||||
IS_TAG=$([ "${{ github.ref_type }}" = "tag" ] && echo 1 || echo 0)
|
||||
REL="${{ github.ref_name }}" # the git tag, verbatim: v1.26.19
|
||||
VER="${REL#v}" # v-stripped alias: 1.26.19
|
||||
yq -o=json -I=0 '.images' hanzo.yml | jq -c '.[]' | while read -r img; do
|
||||
name=$(echo "$img"|jq -r .name); ctx=$(echo "$img"|jq -r .context)
|
||||
df=$(echo "$img"|jq -r '.dockerfile // (.context+"/Dockerfile")'); repo=$(echo "$img"|jq -r .repo)
|
||||
# tag-suffix is OPTIONAL (most repos ship a single variant). When set
|
||||
# (e.g. "ce"/"ee") it qualifies every tag; when absent the tags are
|
||||
# clean (no trailing dash). Build + deploy must agree on this shape.
|
||||
sfx=$(echo "$img"|jq -r '."tag-suffix" // ""')
|
||||
# platforms is OPT-IN per image in hanzo.yml (default: amd64 only, so
|
||||
# every existing repo's tag shape "-amd64" is UNCHANGED). Set e.g.
|
||||
# platforms: [linux/amd64, linux/arm64]
|
||||
# to emit a multi-arch MANIFEST LIST — one digest serving both arches.
|
||||
# DOKS has no arm64 nodes, so arm64 builds via buildx QEMU emulation
|
||||
# (binfmt set up below); pure-Go (CGO_ENABLED=0) Dockerfiles that honor
|
||||
# $TARGETARCH cross-compile natively (fast, no emulation). For true
|
||||
# native-speed arm64, register a bare-metal arm64 host (spark/GB10) as
|
||||
# the hanzo-build-linux-arm64 self-hosted runner (values-build-arm64.yaml).
|
||||
plats=$(echo "$img"|jq -r '(.platforms // ["linux/amd64"]) | join(",")')
|
||||
# ---- Refuse a Go builder older than the module it compiles -------
|
||||
# The official golang images set GOTOOLCHAIN=local, so a go.mod
|
||||
# requiring a newer Go than the base image does not degrade — it
|
||||
# dies mid-build with "go.mod requires go >= X (running Y;
|
||||
# GOTOOLCHAIN=local)". hanzoai/visor v1.108.16 is the shipped
|
||||
# instance, and it is invisible before the image build: every local
|
||||
# `go build` succeeds because a dev box runs GOTOOLCHAIN=auto and
|
||||
# just downloads what the module asks for. So the break is
|
||||
# introduced by editing go.mod — a file with nothing to do with
|
||||
# Docker — and found by a red release build.
|
||||
#
|
||||
# It is not one repo's problem either: a sweep of every Dockerfile
|
||||
# across the orgs found 54 below their own go.mod in 23 repos, and
|
||||
# only 7 of 223 Go builder stages set GOTOOLCHAIN=auto. Fixing those
|
||||
# fixes today; this line is what makes the 55th impossible.
|
||||
#
|
||||
# Checked here, before any build work, so the run fails in seconds
|
||||
# rather than after binfmt, a version derivation and a layer cache.
|
||||
"$CI_HOME/bin/gover" "$df" "$ctx"
|
||||
# ---- The semver this build publishes ----------------------------
|
||||
# A branch build used to publish sha-<short7> and nothing else, so an
|
||||
# image only earned a version when a human remembered to cut a git
|
||||
# tag. That is the whole reason 14 of the fleet's 117 pins named a
|
||||
# commit instead of a release: not drift, just the only tag CI ever
|
||||
# offered them. Every build now publishes a version too, and the
|
||||
# sha- tag stays beside it for forensics.
|
||||
#
|
||||
# The number is DERIVED, never typed, by bin/imgver — the ONE
|
||||
# implementation, shared with the hand-rolled
|
||||
# .hanzo/workflows/deploy.yml that 11 repos carry instead of importing
|
||||
# this workflow (see .github/actions/imgver). Written twice it would be
|
||||
# right twice and then wrong once, which is exactly how `sha-<short>`
|
||||
# became the only tag those repos ever published. Read that script for
|
||||
# the rule; it is monotonic against the repo's manifest AND the
|
||||
# registry, so one tag can never cover two digests.
|
||||
ver=""
|
||||
if [ "$IS_TAG" = 1 ]; then
|
||||
# Release build: the git tag IS the version. Identity in, identity
|
||||
# out — deriving here would publish a number nobody asked for.
|
||||
:
|
||||
else
|
||||
# An explicit `version:` on the image entry wins over the repo-wide
|
||||
# one; both are optional, and imgver falls back to the manifest.
|
||||
iv=$(echo "$img"|jq -r '.version // ""')
|
||||
[ -z "$iv" ] && iv=$(yq -r '.version // ""' hanzo.yml)
|
||||
ver=$(IMGVER_VERSION="$iv" "$CI_HOME/bin/imgver" "$repo" "$ctx")
|
||||
fi
|
||||
if [ "$plats" = "linux/amd64" ]; then
|
||||
# single-arch: keep the exact legacy tag shape (-amd64) deploys expect.
|
||||
TAGS="-t $repo:sha-${SHORT}-amd64${sfx:+-$sfx} -t $repo:${sfx:+$sfx-}latest"
|
||||
else
|
||||
# multi-arch: one arch-neutral manifest-list tag (no -amd64 suffix).
|
||||
docker run --privileged --rm tonistiigi/binfmt --install arm64 >/dev/null 2>&1 || true
|
||||
TAGS="-t $repo:sha-${SHORT}${sfx:+-$sfx} -t $repo:${sfx:+$sfx-}latest"
|
||||
fi
|
||||
# Release (tag) build. The git tag IS the release name, so publish it
|
||||
# VERBATIM (v1.26.19) — that is the shape a universe CR pins, and
|
||||
# stripping the v is why releases were finished by hand-`crane copy`ing
|
||||
# sha-<sha7> onto the semver a human typed. Identity in, identity out.
|
||||
# The v-stripped alias stays for CRs already pinned that way (world
|
||||
# 2.4.51), and is skipped when a repo tags without a v. The old
|
||||
# `<ver>-amd64` alias is deleted: no CR in the fleet pinned it.
|
||||
if [ "$IS_TAG" = 1 ]; then
|
||||
TAGS="$TAGS -t $repo:${REL}${sfx:+-$sfx}"
|
||||
[ "$REL" != "$VER" ] && TAGS="$TAGS -t $repo:${VER}${sfx:+-$sfx}"
|
||||
else
|
||||
# The tag universe is meant to PIN. It goes on every build, not
|
||||
# only the ones somebody remembered to tag by hand.
|
||||
TAGS="$TAGS -t $repo:${ver}${sfx:+-$sfx}"
|
||||
fi
|
||||
echo "::group::build $name → $repo (${sfx}) [$plats]"
|
||||
# --build-arg assembly: static hanzo.yml `args` (a fixed value, e.g. a
|
||||
# pinned base image tag) + `build_secrets` (KMS values the KMS step
|
||||
# exported into the env above). Empty when a repo declares neither, so
|
||||
# the buildx line is unchanged for every existing repo.
|
||||
BUILD_ARGS=""
|
||||
while IFS= read -r kv; do [ -n "$kv" ] && BUILD_ARGS="$BUILD_ARGS --build-arg $kv"; done \
|
||||
< <(echo "$img" | jq -r '(.args // {}) | to_entries[] | "\(.key)=\(.value)"')
|
||||
for bs in $(echo "$img" | jq -r '(.build_secrets // [])[]'); do
|
||||
v=$(printenv "$bs" 2>/dev/null || true)
|
||||
# Defense in depth: the KMS step above already refuses to continue
|
||||
# with an unresolved build_secret, so reaching here empty means the
|
||||
# two steps disagree. Stop rather than bake a blank credential.
|
||||
[ -z "$v" ] && { echo "::error::build_secret $bs is unset at build time — refusing to bake an empty value"; exit 1; }
|
||||
BUILD_ARGS="$BUILD_ARGS --build-arg $bs=$v"
|
||||
done
|
||||
# PROVENANCE: which commit are these bytes. Passed BOTH ways on
|
||||
# purpose. The --label sets it from outside for any Dockerfile; the
|
||||
# --build-arg feeds `ARG REVISION` for the Dockerfiles that stamp the
|
||||
# label themselves (hanzoai/cloud declares `ARG REVISION=unknown`,
|
||||
# and `unknown` is exactly what every image built without this said).
|
||||
#
|
||||
# An unlabelled image cannot be traced to source, and that stops being
|
||||
# cosmetic the moment two builders race for one tag: with one image
|
||||
# labelled and the other reading `unknown`, "which of these is the
|
||||
# release" had no answer short of diffing layers — while the wrong one
|
||||
# was already pinned. A version is a receipt only if the image can
|
||||
# name its own commit, so every image this fleet publishes now can.
|
||||
#
|
||||
# A Dockerfile with no `ARG REVISION` ignores the build-arg (buildkit
|
||||
# warns, never fails), so this is a no-op for every repo that has not
|
||||
# asked for it.
|
||||
PROV="--build-arg REVISION=$GITHUB_SHA"
|
||||
PROV="$PROV --label org.opencontainers.image.revision=$GITHUB_SHA"
|
||||
PROV="$PROV --label org.opencontainers.image.version=${ver:-$REL}"
|
||||
PROV="$PROV --label org.opencontainers.image.source=https://github.com/$GITHUB_REPOSITORY"
|
||||
# GIT_TOKEN (from KMS, via GITHUB_ENV) is passed as the `gh_token`
|
||||
# BuildKit secret so Dockerfiles can clone private Go modules; omitted
|
||||
# cleanly when absent (public-only builds unaffected).
|
||||
docker buildx build --platform "$plats" $BUILD_ARGS $PROV ${GIT_TOKEN:+--secret id=gh_token,env=GIT_TOKEN} --push $TAGS -f "$df" "$ctx"
|
||||
# The semver is the tag the fleet PINS, so prove it exists before the
|
||||
# run goes green — buildx can exit 0 while the manifest is not yet
|
||||
# resolvable, and a pin onto a phantom tag is an ImagePullBackOff.
|
||||
# The digest is printed with it because universe pins repo:tag@digest
|
||||
# and the kubelet honours the DIGEST: publishing both here is what
|
||||
# lets the two move together instead of being looked up separately
|
||||
# and drifting apart.
|
||||
pin="$repo:${ver:-$REL}${sfx:+-$sfx}"
|
||||
for i in 1 2 3 4 5 6; do
|
||||
dig=$(docker buildx imagetools inspect "$pin" --format '{{.Manifest.Digest}}' 2>/dev/null || true)
|
||||
[ -n "$dig" ] && break
|
||||
sleep 5
|
||||
done
|
||||
[ -n "$dig" ] || { echo "::error::$pin not resolvable after push"; exit 1; }
|
||||
echo "| \`$pin\` | \`$dig\` |" >> "$RUNNER_TEMP/pins.md"
|
||||
# Dual-host: mirror the exact tag set to registry.hanzo.ai (server-
|
||||
# side manifest copy — no rebuild). ghcr.io/<org>/<name> →
|
||||
# oci.hanzo.ai/<org>/<name>; public consumers keep ghcr, the fleet
|
||||
# is migrating to pull from ours. A skip here is now a WARNING, not
|
||||
# a notice: an image that never reaches our registry is the reason
|
||||
# a deploy still depends on GitHub, and that should be visible in
|
||||
# the run, not buried.
|
||||
if [ "${MIRROR_OK:-}" = "1" ]; then
|
||||
# crane, not buildx imagetools: the IAM token realm doesn't answer
|
||||
# buildx's multi-scope token request (spec gap, tracked).
|
||||
command -v crane >/dev/null 2>&1 || {
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
curl -fsSL https://github.com/google/go-containerregistry/releases/download/v0.20.2/go-containerregistry_Linux_x86_64.tar.gz \
|
||||
| tar -xz -C "$HOME/.local/bin" crane
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
}
|
||||
mrepo="oci.hanzo.ai/${repo#*/}"
|
||||
echo "$TAGS" | tr ' ' '\n' | grep -v '^-t$' | grep -v '^$' | while read -r ref; do
|
||||
crane copy "$ref" "${mrepo}:${ref##*:}" \
|
||||
|| echo "::warning::$ref did not reach oci.hanzo.ai (ghcr push unaffected)"
|
||||
done
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
done
|
||||
# The pins this run earned, ready to paste into universe. Tag and
|
||||
# digest are printed TOGETHER because they are one fact: the failure
|
||||
# this prevents is a new tag landing beside an old digest, which
|
||||
# reports the new version on the pod and serves the old bytes.
|
||||
if [ -s "$RUNNER_TEMP/pins.md" ]; then
|
||||
{ echo "### Pins"; echo; echo "| image | digest |"; echo "|---|---|";
|
||||
cat "$RUNNER_TEMP/pins.md"; } >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
|
||||
- name: Provision Go toolchain (go test gates on bare runners)
|
||||
# hanzo.yml `test:` gates (e.g. `go vet ./...`, `go test ...`) run
|
||||
# DIRECTLY on the runner, NOT inside a build container — but the stock
|
||||
@@ -1013,16 +839,72 @@ jobs:
|
||||
# unset) — so `go vet` dies with `could not read Username for github.com`.
|
||||
# Reuse the SAME token here for the runner's git (GIT_TOKEN, set by the
|
||||
# KMS step above; GH_PAT fallback). No-op when neither is present.
|
||||
#
|
||||
# AND OUR OWN MODULES RESOLVE FROM OUR OWN FORGE. A module path is a NAME,
|
||||
# not an address: `github.com/<owner>/x` is what the package is called, and
|
||||
# which host git dials for it is a separate decision. On a forge runner
|
||||
# that decision is the forge — the forge is where the code canonically
|
||||
# lives, and github.com is fed by a mirror that trails it, sometimes by
|
||||
# hours and sometimes not at all.
|
||||
#
|
||||
# Measured, on the first run in which this step's gate had ever actually
|
||||
# executed: hanzoai/cloud's go.mod carries `github.com/hanzoai/dashscopego
|
||||
# v0.6.0` as an indirect dependency. git.hanzo.ai serves it; github.com
|
||||
# answers `Repository not found`. So `go vet` died before a single test
|
||||
# ran, on a module that was never missing. Its own caller already knew the
|
||||
# answer — cloud's `containment` and `reach` jobs each make this exact
|
||||
# substitution, with a comment about the nine consecutive releases an
|
||||
# ACL drift on ONE sibling module once blocked — but the gate that runs
|
||||
# its tests reached the modules through a different door.
|
||||
#
|
||||
# go.sum is what makes this safe rather than merely convenient: the forge
|
||||
# serves the same objects, so the fetched zip hashes to the h1: line
|
||||
# already committed. A forge serving different bytes fails the build
|
||||
# loudly instead of shipping them.
|
||||
#
|
||||
# Longest prefix wins in git, so the owner-scoped forge rule outranks the
|
||||
# bare github.com rule beneath it and every OTHER module still resolves
|
||||
# from github.com. Both rules are optional and independent: no FORGE_TOKEN
|
||||
# (or a github.com-hosted run, where the server IS github.com) leaves the
|
||||
# old behaviour exactly as it was.
|
||||
#
|
||||
# SPLIT THE SERVER, do not strip a scheme you assumed. GITHUB_SERVER_URL
|
||||
# on this fleet's runners is `http://hanzo-git.hanzo.svc` — the forge's
|
||||
# in-cluster address, plain http — and a `${SERVER#https://}` that only
|
||||
# knows the https spelling passes it through whole, producing
|
||||
# `https://http://hanzo-git.hanzo.svc/...` and a git that resolves the host
|
||||
# `http`. Take the scheme and the host as the two values they are, and
|
||||
# compare the HOST against github.com so the guard cannot be fooled by a
|
||||
# spelling either.
|
||||
if: inputs.mode != 'delegate' && hashFiles('go.mod') != ''
|
||||
env:
|
||||
GH_PAT: ${{ secrets.GH_PAT }}
|
||||
FORGE_TOKEN: ${{ secrets.FORGE_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
TOKEN="${GIT_TOKEN:-${GH_PAT:-}}"
|
||||
if [ -z "$TOKEN" ]; then echo "no git token — public modules only"; exit 0; fi
|
||||
CFG="$RUNNER_TEMP/gitconfig-go-test"; : > "$CFG"
|
||||
GIT_CONFIG_GLOBAL="$CFG" git config --global \
|
||||
url."https://x-access-token:${TOKEN}@github.com/".insteadOf "https://github.com/"
|
||||
SERVER="${GITHUB_SERVER_URL:-https://github.com}"
|
||||
SCHEME="${SERVER%%://*}"; HOST="${SERVER#*://}"; HOST="${HOST%/}"
|
||||
wrote=0
|
||||
|
||||
if [ -n "${FORGE_TOKEN:-}" ] && [ "$HOST" != "github.com" ]; then
|
||||
GIT_CONFIG_GLOBAL="$CFG" git config --global \
|
||||
url."${SCHEME}://x:${FORGE_TOKEN}@${HOST}/${OWNER}/".insteadOf \
|
||||
"https://github.com/${OWNER}/"
|
||||
echo "${OWNER}/* modules resolve from ${SCHEME}://${HOST}"
|
||||
wrote=1
|
||||
fi
|
||||
|
||||
TOKEN="${GIT_TOKEN:-${GH_PAT:-}}"
|
||||
if [ -n "$TOKEN" ]; then
|
||||
GIT_CONFIG_GLOBAL="$CFG" git config --global \
|
||||
url."https://x-access-token:${TOKEN}@github.com/".insteadOf "https://github.com/"
|
||||
echo "github.com modules authenticated"
|
||||
wrote=1
|
||||
fi
|
||||
|
||||
[ "$wrote" = 1 ] || { echo "no git credential — public modules only"; exit 0; }
|
||||
{ echo "GIT_CONFIG_GLOBAL=$CFG"; echo "GIT_CONFIG_NOSYSTEM=1"; } >> "$GITHUB_ENV"
|
||||
echo "runner git authenticated for private Go modules"
|
||||
|
||||
@@ -1074,6 +956,227 @@ jobs:
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Build & push images (per hanzo.yml)
|
||||
# AFTER the gate, deliberately. This step used to run before `test:`,
|
||||
# so a repo whose gates FAILED still published an image: hanzoai/base
|
||||
# f9ec7067 pushed sha-f9ec706-amd64 on a run whose tests then went red.
|
||||
# Deploy and Publish site were skipped, so nothing shipped it — but the
|
||||
# tag existed, and a tag that exists is something a human pins by hand.
|
||||
# An artifact is a claim that the commit passed; it should not be
|
||||
# possible to produce one from a commit that did not.
|
||||
#
|
||||
# Nothing here needs the toolchains above it, and Deploy still follows
|
||||
# this step, so the only thing the move changes is that a red gate now
|
||||
# produces no image at all.
|
||||
if: inputs.mode != 'delegate'
|
||||
env:
|
||||
GH_PAT: ${{ secrets.GH_PAT }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Test-only callers (hanzo.yml without `images:` — e.g. a repo whose
|
||||
# image lane lives in its own release.yml, or a pure library) skip the
|
||||
# build step entirely instead of exploding on a null .images.
|
||||
if [ "$(yq -r '.images // [] | length' hanzo.yml 2>/dev/null || echo 0)" = "0" ]; then
|
||||
echo "::notice::no images: in hanzo.yml — test-only caller, skipping build"; exit 0
|
||||
fi
|
||||
# Build-time private cross-org Go module read (the buildx `gh_token`
|
||||
# secret): prefer the KMS-fetched GIT_TOKEN, else fall back to the org
|
||||
# GH_PAT — the SAME BuildKit gh_token cloud's release.yml uses (proven
|
||||
# working). Keeps image builds green when the KMS deploy-cred fetch is
|
||||
# unavailable (a repo with a private cross-org dep like hanzoai/cloud
|
||||
# otherwise fails `go mod tidy` with git exit 128 in the buildx stage).
|
||||
# No-op for public-only builds when both are empty. Exported so the
|
||||
# `--secret id=gh_token,env=GIT_TOKEN` below reads it from the env.
|
||||
export GIT_TOKEN="${GIT_TOKEN:-${GH_PAT:-}}"
|
||||
SHORT=$(echo "${{ github.sha }}" | cut -c1-7)
|
||||
IS_TAG=$([ "${{ github.ref_type }}" = "tag" ] && echo 1 || echo 0)
|
||||
REL="${{ github.ref_name }}" # the git tag, verbatim: v1.26.19
|
||||
VER="${REL#v}" # v-stripped alias: 1.26.19
|
||||
yq -o=json -I=0 '.images' hanzo.yml | jq -c '.[]' | while read -r img; do
|
||||
name=$(echo "$img"|jq -r .name); ctx=$(echo "$img"|jq -r .context)
|
||||
df=$(echo "$img"|jq -r '.dockerfile // (.context+"/Dockerfile")'); repo=$(echo "$img"|jq -r .repo)
|
||||
# tag-suffix is OPTIONAL (most repos ship a single variant). When set
|
||||
# (e.g. "ce"/"ee") it qualifies every tag; when absent the tags are
|
||||
# clean (no trailing dash). Build + deploy must agree on this shape.
|
||||
sfx=$(echo "$img"|jq -r '."tag-suffix" // ""')
|
||||
# platforms is OPT-IN per image in hanzo.yml (default: amd64 only, so
|
||||
# every existing repo's tag shape "-amd64" is UNCHANGED). Set e.g.
|
||||
# platforms: [linux/amd64, linux/arm64]
|
||||
# to emit a multi-arch MANIFEST LIST — one digest serving both arches.
|
||||
# DOKS has no arm64 nodes, so arm64 builds via buildx QEMU emulation
|
||||
# (binfmt set up below); pure-Go (CGO_ENABLED=0) Dockerfiles that honor
|
||||
# $TARGETARCH cross-compile natively (fast, no emulation). For true
|
||||
# native-speed arm64, register a bare-metal arm64 host (spark/GB10) as
|
||||
# the hanzo-build-linux-arm64 self-hosted runner (values-build-arm64.yaml).
|
||||
plats=$(echo "$img"|jq -r '(.platforms // ["linux/amd64"]) | join(",")')
|
||||
# ---- Refuse a Go builder older than the module it compiles -------
|
||||
# The official golang images set GOTOOLCHAIN=local, so a go.mod
|
||||
# requiring a newer Go than the base image does not degrade — it
|
||||
# dies mid-build with "go.mod requires go >= X (running Y;
|
||||
# GOTOOLCHAIN=local)". hanzoai/visor v1.108.16 is the shipped
|
||||
# instance, and it is invisible before the image build: every local
|
||||
# `go build` succeeds because a dev box runs GOTOOLCHAIN=auto and
|
||||
# just downloads what the module asks for. So the break is
|
||||
# introduced by editing go.mod — a file with nothing to do with
|
||||
# Docker — and found by a red release build.
|
||||
#
|
||||
# It is not one repo's problem either: a sweep of every Dockerfile
|
||||
# across the orgs found 54 below their own go.mod in 23 repos, and
|
||||
# only 7 of 223 Go builder stages set GOTOOLCHAIN=auto. Fixing those
|
||||
# fixes today; this line is what makes the 55th impossible.
|
||||
#
|
||||
# Checked here, before any build work, so the run fails in seconds
|
||||
# rather than after binfmt, a version derivation and a layer cache.
|
||||
"$CI_HOME/bin/gover" "$df" "$ctx"
|
||||
# ---- The semver this build publishes ----------------------------
|
||||
# A branch build used to publish sha-<short7> and nothing else, so an
|
||||
# image only earned a version when a human remembered to cut a git
|
||||
# tag. That is the whole reason 14 of the fleet's 117 pins named a
|
||||
# commit instead of a release: not drift, just the only tag CI ever
|
||||
# offered them. Every build now publishes a version too, and the
|
||||
# sha- tag stays beside it for forensics.
|
||||
#
|
||||
# The number is DERIVED, never typed, by bin/imgver — the ONE
|
||||
# implementation, shared with the hand-rolled
|
||||
# .hanzo/workflows/deploy.yml that 11 repos carry instead of importing
|
||||
# this workflow (see .github/actions/imgver). Written twice it would be
|
||||
# right twice and then wrong once, which is exactly how `sha-<short>`
|
||||
# became the only tag those repos ever published. Read that script for
|
||||
# the rule; it is monotonic against the repo's manifest AND the
|
||||
# registry, so one tag can never cover two digests.
|
||||
ver=""
|
||||
if [ "$IS_TAG" = 1 ]; then
|
||||
# Release build: the git tag IS the version. Identity in, identity
|
||||
# out — deriving here would publish a number nobody asked for.
|
||||
:
|
||||
else
|
||||
# An explicit `version:` on the image entry wins over the repo-wide
|
||||
# one; both are optional, and imgver falls back to the manifest.
|
||||
iv=$(echo "$img"|jq -r '.version // ""')
|
||||
[ -z "$iv" ] && iv=$(yq -r '.version // ""' hanzo.yml)
|
||||
ver=$(IMGVER_VERSION="$iv" "$CI_HOME/bin/imgver" "$repo" "$ctx")
|
||||
fi
|
||||
if [ "$plats" = "linux/amd64" ]; then
|
||||
# single-arch: keep the exact legacy tag shape (-amd64) deploys expect.
|
||||
TAGS="-t $repo:sha-${SHORT}-amd64${sfx:+-$sfx} -t $repo:${sfx:+$sfx-}latest"
|
||||
else
|
||||
# multi-arch: one arch-neutral manifest-list tag (no -amd64 suffix).
|
||||
docker run --privileged --rm tonistiigi/binfmt --install arm64 >/dev/null 2>&1 || true
|
||||
TAGS="-t $repo:sha-${SHORT}${sfx:+-$sfx} -t $repo:${sfx:+$sfx-}latest"
|
||||
fi
|
||||
# Release (tag) build. The git tag IS the release name, so publish it
|
||||
# VERBATIM (v1.26.19) — that is the shape a universe CR pins, and
|
||||
# stripping the v is why releases were finished by hand-`crane copy`ing
|
||||
# sha-<sha7> onto the semver a human typed. Identity in, identity out.
|
||||
# The v-stripped alias stays for CRs already pinned that way (world
|
||||
# 2.4.51), and is skipped when a repo tags without a v. The old
|
||||
# `<ver>-amd64` alias is deleted: no CR in the fleet pinned it.
|
||||
if [ "$IS_TAG" = 1 ]; then
|
||||
TAGS="$TAGS -t $repo:${REL}${sfx:+-$sfx}"
|
||||
[ "$REL" != "$VER" ] && TAGS="$TAGS -t $repo:${VER}${sfx:+-$sfx}"
|
||||
else
|
||||
# The tag universe is meant to PIN. It goes on every build, not
|
||||
# only the ones somebody remembered to tag by hand.
|
||||
TAGS="$TAGS -t $repo:${ver}${sfx:+-$sfx}"
|
||||
fi
|
||||
echo "::group::build $name → $repo (${sfx}) [$plats]"
|
||||
# --build-arg assembly: static hanzo.yml `args` (a fixed value, e.g. a
|
||||
# pinned base image tag) + `build_secrets` (KMS values the KMS step
|
||||
# exported into the env above). Empty when a repo declares neither, so
|
||||
# the buildx line is unchanged for every existing repo.
|
||||
BUILD_ARGS=""
|
||||
while IFS= read -r kv; do [ -n "$kv" ] && BUILD_ARGS="$BUILD_ARGS --build-arg $kv"; done \
|
||||
< <(echo "$img" | jq -r '(.args // {}) | to_entries[] | "\(.key)=\(.value)"')
|
||||
for bs in $(echo "$img" | jq -r '(.build_secrets // [])[]'); do
|
||||
v=$(printenv "$bs" 2>/dev/null || true)
|
||||
# Defense in depth: the KMS step above already refuses to continue
|
||||
# with an unresolved build_secret, so reaching here empty means the
|
||||
# two steps disagree. Stop rather than bake a blank credential.
|
||||
[ -z "$v" ] && { echo "::error::build_secret $bs is unset at build time — refusing to bake an empty value"; exit 1; }
|
||||
BUILD_ARGS="$BUILD_ARGS --build-arg $bs=$v"
|
||||
done
|
||||
# PROVENANCE: which commit are these bytes. Passed BOTH ways on
|
||||
# purpose. The --label sets it from outside for any Dockerfile; the
|
||||
# --build-arg feeds `ARG REVISION` for the Dockerfiles that stamp the
|
||||
# label themselves (hanzoai/cloud declares `ARG REVISION=unknown`,
|
||||
# and `unknown` is exactly what every image built without this said).
|
||||
#
|
||||
# An unlabelled image cannot be traced to source, and that stops being
|
||||
# cosmetic the moment two builders race for one tag: with one image
|
||||
# labelled and the other reading `unknown`, "which of these is the
|
||||
# release" had no answer short of diffing layers — while the wrong one
|
||||
# was already pinned. A version is a receipt only if the image can
|
||||
# name its own commit, so every image this fleet publishes now can.
|
||||
#
|
||||
# A Dockerfile with no `ARG REVISION` ignores the build-arg (buildkit
|
||||
# warns, never fails), so this is a no-op for every repo that has not
|
||||
# asked for it.
|
||||
PROV="--build-arg REVISION=$GITHUB_SHA"
|
||||
PROV="$PROV --label org.opencontainers.image.revision=$GITHUB_SHA"
|
||||
PROV="$PROV --label org.opencontainers.image.version=${ver:-$REL}"
|
||||
PROV="$PROV --label org.opencontainers.image.source=https://github.com/$GITHUB_REPOSITORY"
|
||||
# GIT_TOKEN (from KMS, via GITHUB_ENV) is passed as the `gh_token`
|
||||
# BuildKit secret so Dockerfiles can clone private Go modules; omitted
|
||||
# cleanly when absent (public-only builds unaffected).
|
||||
docker buildx build --platform "$plats" $BUILD_ARGS $PROV ${GIT_TOKEN:+--secret id=gh_token,env=GIT_TOKEN} --push $TAGS -f "$df" "$ctx"
|
||||
# The semver is the tag the fleet PINS, so prove it exists before the
|
||||
# run goes green — buildx can exit 0 while the manifest is not yet
|
||||
# resolvable, and a pin onto a phantom tag is an ImagePullBackOff.
|
||||
# The digest is printed with it because universe pins repo:tag@digest
|
||||
# and the kubelet honours the DIGEST: publishing both here is what
|
||||
# lets the two move together instead of being looked up separately
|
||||
# and drifting apart.
|
||||
pin="$repo:${ver:-$REL}${sfx:+-$sfx}"
|
||||
for i in 1 2 3 4 5 6; do
|
||||
dig=$(docker buildx imagetools inspect "$pin" --format '{{.Manifest.Digest}}' 2>/dev/null || true)
|
||||
[ -n "$dig" ] && break
|
||||
sleep 5
|
||||
done
|
||||
[ -n "$dig" ] || { echo "::error::$pin not resolvable after push"; exit 1; }
|
||||
echo "| \`$pin\` | \`$dig\` |" >> "$RUNNER_TEMP/pins.md"
|
||||
# Dual-host: mirror the exact tag set to registry.hanzo.ai (server-
|
||||
# side manifest copy — no rebuild). ghcr.io/<org>/<name> →
|
||||
# registry.hanzo.ai/<org>/<name>; public consumers keep ghcr, the fleet
|
||||
# is migrating to pull from ours. A skip here is now a WARNING, not
|
||||
# a notice: an image that never reaches our registry is the reason
|
||||
# a deploy still depends on GitHub, and that should be visible in
|
||||
# the run, not buried.
|
||||
if [ "${MIRROR_OK:-}" = "1" ]; then
|
||||
# crane, not buildx imagetools: the IAM token realm doesn't answer
|
||||
# buildx's multi-scope token request (spec gap, tracked).
|
||||
command -v crane >/dev/null 2>&1 || {
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
curl -fsSL https://github.com/google/go-containerregistry/releases/download/v0.20.2/go-containerregistry_Linux_x86_64.tar.gz \
|
||||
| tar -xz -C "$HOME/.local/bin" crane
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
}
|
||||
# ONE mapping, stated once. The GHCR org and the registry org are
|
||||
# not the same word — ghcr.io/hanzoai/x mirrors to
|
||||
# registry.hanzo.ai/hanzo/x — and a second copy of this table is
|
||||
# how a push path and a pin drift into naming different images.
|
||||
rorg="${repo#*/}"; rname="${rorg#*/}"; rorg="${rorg%%/*}"
|
||||
case "$rorg" in
|
||||
hanzoai) rorg=hanzo ;;
|
||||
zooai) rorg=zoo ;;
|
||||
*) ;; # bootnode, lux-private, ... map to themselves
|
||||
esac
|
||||
mrepo="registry.hanzo.ai/$rorg/$rname"
|
||||
echo "$TAGS" | tr ' ' '\n' | grep -v '^-t$' | grep -v '^$' | while read -r ref; do
|
||||
crane copy "$ref" "${mrepo}:${ref##*:}" \
|
||||
|| echo "::warning::$ref did not reach registry.hanzo.ai (ghcr push unaffected)"
|
||||
done
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
done
|
||||
# The pins this run earned, ready to paste into universe. Tag and
|
||||
# digest are printed TOGETHER because they are one fact: the failure
|
||||
# this prevents is a new tag landing beside an old digest, which
|
||||
# reports the new version on the pod and serves the old bytes.
|
||||
if [ -s "$RUNNER_TEMP/pins.md" ]; then
|
||||
{ echo "### Pins"; echo; echo "| image | digest |"; echo "|---|---|";
|
||||
cat "$RUNNER_TEMP/pins.md"; } >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
|
||||
- name: Client — commit the projection and cut the patch
|
||||
# LAST, and after `test:` on purpose: this pushes a tag, the tag starts a
|
||||
# publish, and a published client is the one artifact in this workflow
|
||||
|
||||
+316
-213
@@ -73,7 +73,7 @@ on:
|
||||
A string sidesteps it on both planes with identical meaning, and the
|
||||
comparison is against 'false' rather than a truthiness test so that the
|
||||
expression callers already write — hanzoai/cloud's
|
||||
`tests: ${{ github.ref_type != 'tag' }}`, which renders the strings
|
||||
`tests:` set from `github.ref_type != 'tag'`, which renders the strings
|
||||
"true"/"false" — keeps working unchanged.
|
||||
type: string
|
||||
default: 'true'
|
||||
@@ -104,6 +104,14 @@ permissions:
|
||||
jobs:
|
||||
cicd:
|
||||
runs-on: ${{ fromJson(inputs.runner) }}
|
||||
env:
|
||||
# corepack fetches the pnpm/yarn TARBALL ITSELF from registry.npmjs.org,
|
||||
# and it does not read .npmrc to do it — so 317 repos that pin
|
||||
# `packageManager` reach out to npmjs on every install no matter how
|
||||
# carefully their registry is configured. This is the one variable that
|
||||
# closes that hole; without it "installs go through our registry" is true
|
||||
# of the dependencies and false of the package manager fetching them.
|
||||
COREPACK_NPM_REGISTRY: https://pkg.hanzo.ai
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
@@ -427,6 +435,18 @@ jobs:
|
||||
# ask KMS twice.
|
||||
SITE_DECL="$(yq -r '.site | type' hanzo.yml 2>/dev/null || echo '!!null')"
|
||||
if [ -n "$(yq -r '.bucket // ""' hanzo.yml 2>/dev/null || true)" ] || [ "$SITE_DECL" = '!!map' ]; then
|
||||
# The registry credential comes from KMS like every other secret —
|
||||
# the ONLY things that live in GitHub are KMS_CLIENT_ID/SECRET.
|
||||
# Named here so a repo with no kubeconfig (most of them) can still
|
||||
# reach registry.hanzo.ai; the kubeconfig read below stays as the
|
||||
# fallback for repos provisioned before these names existed.
|
||||
for k in REGISTRY_USER REGISTRY_PASSWORD; do
|
||||
v=$(get "$k")
|
||||
if [ -n "$v" ]; then
|
||||
[ "$k" = REGISTRY_PASSWORD ] && echo "::add-mask::$v"
|
||||
echo "$k=$v" >> "$GITHUB_ENV"
|
||||
fi
|
||||
done
|
||||
for k in S3_ADMIN_ACCESS_KEY S3_ADMIN_SECRET_KEY; do
|
||||
v=$(get "$k")
|
||||
if [ -n "$v" ]; then echo "::add-mask::$v"; echo "$k=$v" >> "$GITHUB_ENV"; fi
|
||||
@@ -521,14 +541,20 @@ jobs:
|
||||
echo "::notice::KMS GHCR token login failed — keeping the earlier login"
|
||||
fi
|
||||
|
||||
- name: Native registry credential (oci.hanzo.ai)
|
||||
- name: Native registry credential (registry.hanzo.ai)
|
||||
if: inputs.mode != 'delegate'
|
||||
env:
|
||||
KUBECONFIG: ${{ steps.kms.outputs.kubeconfig }}
|
||||
REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
|
||||
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
# KMS first (exported to GITHUB_ENV by the step above), GitHub secret
|
||||
# second. `env:` here cannot see GITHUB_ENV from THIS step's own env
|
||||
# block, so the KMS values are read out of the environment in `run:`.
|
||||
GH_REGISTRY_USER: ${{ secrets.REGISTRY_USER }}
|
||||
GH_REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
# REGISTRY_USER/PASSWORD may already be in the environment from KMS.
|
||||
REGISTRY_USER="${REGISTRY_USER:-${GH_REGISTRY_USER:-}}"
|
||||
REGISTRY_PASSWORD="${REGISTRY_PASSWORD:-${GH_REGISTRY_PASSWORD:-}}"
|
||||
# Direct credential first (repo/org secret — works on private repos,
|
||||
# where the Free plan hides org secrets... including these; private
|
||||
# repos set them at REPO level). KMS-kubeconfig read is the fallback.
|
||||
@@ -537,14 +563,14 @@ jobs:
|
||||
# cred) must never fail the run — the image still pushes to GHCR, the
|
||||
# primary. Without this guard, bash -e aborts the step and SKIPS the
|
||||
# build entirely (a registry hiccup takes the whole lane red).
|
||||
if echo "$REGISTRY_PASSWORD" | docker login oci.hanzo.ai -u "$REGISTRY_USER" --password-stdin; then
|
||||
if echo "$REGISTRY_PASSWORD" | docker login registry.hanzo.ai -u "$REGISTRY_USER" --password-stdin; then
|
||||
echo "MIRROR_OK=1" >> "$GITHUB_ENV"
|
||||
else
|
||||
echo "::notice::registry.hanzo.ai login failed — mirror skipped (GHCR-only push)"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
[ -z "${KUBECONFIG:-}" ] && { echo "::warning::no registry credential and no kubeconfig — image will NOT reach oci.hanzo.ai (ghcr only)"; exit 0; }
|
||||
[ -z "${KUBECONFIG:-}" ] && { echo "::warning::no registry credential and no kubeconfig — image will NOT reach registry.hanzo.ai (ghcr only)"; exit 0; }
|
||||
# Bare runners ship no kubectl — same static provision the deploy
|
||||
# step uses.
|
||||
command -v kubectl >/dev/null 2>&1 || {
|
||||
@@ -555,218 +581,18 @@ jobs:
|
||||
}
|
||||
CFG=$(kubectl -n hanzo get secret registry-credentials -o jsonpath='{.data.\.dockerconfigjson}' 2>/dev/null | base64 -d || true)
|
||||
if [ -z "$CFG" ]; then
|
||||
echo "::warning::registry-credentials not readable from this kubeconfig — image will NOT reach oci.hanzo.ai (ghcr only)"; exit 0
|
||||
echo "::warning::registry-credentials not readable from this kubeconfig — image will NOT reach registry.hanzo.ai (ghcr only)"; exit 0
|
||||
fi
|
||||
USERPASS=$(echo "$CFG" | jq -r '.auths["registry.hanzo.ai"].auth // empty' | base64 -d)
|
||||
[ -z "$USERPASS" ] && { echo "::warning::no registry auth in dockerconfig — image will NOT reach oci.hanzo.ai (ghcr only)"; exit 0; }
|
||||
[ -z "$USERPASS" ] && { echo "::warning::no registry auth in dockerconfig — image will NOT reach registry.hanzo.ai (ghcr only)"; exit 0; }
|
||||
echo "::add-mask::${USERPASS#*:}"
|
||||
# Best-effort: login failure → skip mirror, never fail the run (see above).
|
||||
if echo "${USERPASS#*:}" | docker login oci.hanzo.ai -u "${USERPASS%%:*}" --password-stdin; then
|
||||
if echo "${USERPASS#*:}" | docker login registry.hanzo.ai -u "${USERPASS%%:*}" --password-stdin; then
|
||||
echo "MIRROR_OK=1" >> "$GITHUB_ENV"
|
||||
else
|
||||
echo "::notice::registry.hanzo.ai login failed — mirror skipped (GHCR-only push)"
|
||||
fi
|
||||
|
||||
- name: Build & push images (per hanzo.yml)
|
||||
if: inputs.mode != 'delegate'
|
||||
env:
|
||||
GH_PAT: ${{ secrets.GH_PAT }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Test-only callers (hanzo.yml without `images:` — e.g. a repo whose
|
||||
# image lane lives in its own release.yml, or a pure library) skip the
|
||||
# build step entirely instead of exploding on a null .images.
|
||||
if [ "$(yq -r '.images // [] | length' hanzo.yml 2>/dev/null || echo 0)" = "0" ]; then
|
||||
echo "::notice::no images: in hanzo.yml — test-only caller, skipping build"; exit 0
|
||||
fi
|
||||
# Build-time private cross-org Go module read (the buildx `gh_token`
|
||||
# secret): prefer the KMS-fetched GIT_TOKEN, else fall back to the org
|
||||
# GH_PAT — the SAME BuildKit gh_token cloud's release.yml uses (proven
|
||||
# working). Keeps image builds green when the KMS deploy-cred fetch is
|
||||
# unavailable (a repo with a private cross-org dep like hanzoai/cloud
|
||||
# otherwise fails `go mod tidy` with git exit 128 in the buildx stage).
|
||||
# No-op for public-only builds when both are empty. Exported so the
|
||||
# `--secret id=gh_token,env=GIT_TOKEN` below reads it from the env.
|
||||
export GIT_TOKEN="${GIT_TOKEN:-${GH_PAT:-}}"
|
||||
SHORT=$(echo "${{ github.sha }}" | cut -c1-7)
|
||||
IS_TAG=$([ "${{ github.ref_type }}" = "tag" ] && echo 1 || echo 0)
|
||||
REL="${{ github.ref_name }}" # the git tag, verbatim: v1.26.19
|
||||
VER="${REL#v}" # v-stripped alias: 1.26.19
|
||||
yq -o=json -I=0 '.images' hanzo.yml | jq -c '.[]' | while read -r img; do
|
||||
name=$(echo "$img"|jq -r .name); ctx=$(echo "$img"|jq -r .context)
|
||||
df=$(echo "$img"|jq -r '.dockerfile // (.context+"/Dockerfile")'); repo=$(echo "$img"|jq -r .repo)
|
||||
# tag-suffix is OPTIONAL (most repos ship a single variant). When set
|
||||
# (e.g. "ce"/"ee") it qualifies every tag; when absent the tags are
|
||||
# clean (no trailing dash). Build + deploy must agree on this shape.
|
||||
sfx=$(echo "$img"|jq -r '."tag-suffix" // ""')
|
||||
# platforms is OPT-IN per image in hanzo.yml (default: amd64 only, so
|
||||
# every existing repo's tag shape "-amd64" is UNCHANGED). Set e.g.
|
||||
# platforms: [linux/amd64, linux/arm64]
|
||||
# to emit a multi-arch MANIFEST LIST — one digest serving both arches.
|
||||
# DOKS has no arm64 nodes, so arm64 builds via buildx QEMU emulation
|
||||
# (binfmt set up below); pure-Go (CGO_ENABLED=0) Dockerfiles that honor
|
||||
# $TARGETARCH cross-compile natively (fast, no emulation). For true
|
||||
# native-speed arm64, register a bare-metal arm64 host (spark/GB10) as
|
||||
# the hanzo-build-linux-arm64 self-hosted runner (values-build-arm64.yaml).
|
||||
plats=$(echo "$img"|jq -r '(.platforms // ["linux/amd64"]) | join(",")')
|
||||
# ---- Refuse a Go builder older than the module it compiles -------
|
||||
# The official golang images set GOTOOLCHAIN=local, so a go.mod
|
||||
# requiring a newer Go than the base image does not degrade — it
|
||||
# dies mid-build with "go.mod requires go >= X (running Y;
|
||||
# GOTOOLCHAIN=local)". hanzoai/visor v1.108.16 is the shipped
|
||||
# instance, and it is invisible before the image build: every local
|
||||
# `go build` succeeds because a dev box runs GOTOOLCHAIN=auto and
|
||||
# just downloads what the module asks for. So the break is
|
||||
# introduced by editing go.mod — a file with nothing to do with
|
||||
# Docker — and found by a red release build.
|
||||
#
|
||||
# It is not one repo's problem either: a sweep of every Dockerfile
|
||||
# across the orgs found 54 below their own go.mod in 23 repos, and
|
||||
# only 7 of 223 Go builder stages set GOTOOLCHAIN=auto. Fixing those
|
||||
# fixes today; this line is what makes the 55th impossible.
|
||||
#
|
||||
# Checked here, before any build work, so the run fails in seconds
|
||||
# rather than after binfmt, a version derivation and a layer cache.
|
||||
"$CI_HOME/bin/gover" "$df" "$ctx"
|
||||
# ---- The semver this build publishes ----------------------------
|
||||
# A branch build used to publish sha-<short7> and nothing else, so an
|
||||
# image only earned a version when a human remembered to cut a git
|
||||
# tag. That is the whole reason 14 of the fleet's 117 pins named a
|
||||
# commit instead of a release: not drift, just the only tag CI ever
|
||||
# offered them. Every build now publishes a version too, and the
|
||||
# sha- tag stays beside it for forensics.
|
||||
#
|
||||
# The number is DERIVED, never typed, by bin/imgver — the ONE
|
||||
# implementation, shared with the hand-rolled
|
||||
# .hanzo/workflows/deploy.yml that 11 repos carry instead of importing
|
||||
# this workflow (see .github/actions/imgver). Written twice it would be
|
||||
# right twice and then wrong once, which is exactly how `sha-<short>`
|
||||
# became the only tag those repos ever published. Read that script for
|
||||
# the rule; it is monotonic against the repo's manifest AND the
|
||||
# registry, so one tag can never cover two digests.
|
||||
ver=""
|
||||
if [ "$IS_TAG" = 1 ]; then
|
||||
# Release build: the git tag IS the version. Identity in, identity
|
||||
# out — deriving here would publish a number nobody asked for.
|
||||
:
|
||||
else
|
||||
# An explicit `version:` on the image entry wins over the repo-wide
|
||||
# one; both are optional, and imgver falls back to the manifest.
|
||||
iv=$(echo "$img"|jq -r '.version // ""')
|
||||
[ -z "$iv" ] && iv=$(yq -r '.version // ""' hanzo.yml)
|
||||
ver=$(IMGVER_VERSION="$iv" "$CI_HOME/bin/imgver" "$repo" "$ctx")
|
||||
fi
|
||||
if [ "$plats" = "linux/amd64" ]; then
|
||||
# single-arch: keep the exact legacy tag shape (-amd64) deploys expect.
|
||||
TAGS="-t $repo:sha-${SHORT}-amd64${sfx:+-$sfx} -t $repo:${sfx:+$sfx-}latest"
|
||||
else
|
||||
# multi-arch: one arch-neutral manifest-list tag (no -amd64 suffix).
|
||||
docker run --privileged --rm tonistiigi/binfmt --install arm64 >/dev/null 2>&1 || true
|
||||
TAGS="-t $repo:sha-${SHORT}${sfx:+-$sfx} -t $repo:${sfx:+$sfx-}latest"
|
||||
fi
|
||||
# Release (tag) build. The git tag IS the release name, so publish it
|
||||
# VERBATIM (v1.26.19) — that is the shape a universe CR pins, and
|
||||
# stripping the v is why releases were finished by hand-`crane copy`ing
|
||||
# sha-<sha7> onto the semver a human typed. Identity in, identity out.
|
||||
# The v-stripped alias stays for CRs already pinned that way (world
|
||||
# 2.4.51), and is skipped when a repo tags without a v. The old
|
||||
# `<ver>-amd64` alias is deleted: no CR in the fleet pinned it.
|
||||
if [ "$IS_TAG" = 1 ]; then
|
||||
TAGS="$TAGS -t $repo:${REL}${sfx:+-$sfx}"
|
||||
[ "$REL" != "$VER" ] && TAGS="$TAGS -t $repo:${VER}${sfx:+-$sfx}"
|
||||
else
|
||||
# The tag universe is meant to PIN. It goes on every build, not
|
||||
# only the ones somebody remembered to tag by hand.
|
||||
TAGS="$TAGS -t $repo:${ver}${sfx:+-$sfx}"
|
||||
fi
|
||||
echo "::group::build $name → $repo (${sfx}) [$plats]"
|
||||
# --build-arg assembly: static hanzo.yml `args` (a fixed value, e.g. a
|
||||
# pinned base image tag) + `build_secrets` (KMS values the KMS step
|
||||
# exported into the env above). Empty when a repo declares neither, so
|
||||
# the buildx line is unchanged for every existing repo.
|
||||
BUILD_ARGS=""
|
||||
while IFS= read -r kv; do [ -n "$kv" ] && BUILD_ARGS="$BUILD_ARGS --build-arg $kv"; done \
|
||||
< <(echo "$img" | jq -r '(.args // {}) | to_entries[] | "\(.key)=\(.value)"')
|
||||
for bs in $(echo "$img" | jq -r '(.build_secrets // [])[]'); do
|
||||
v=$(printenv "$bs" 2>/dev/null || true)
|
||||
# Defense in depth: the KMS step above already refuses to continue
|
||||
# with an unresolved build_secret, so reaching here empty means the
|
||||
# two steps disagree. Stop rather than bake a blank credential.
|
||||
[ -z "$v" ] && { echo "::error::build_secret $bs is unset at build time — refusing to bake an empty value"; exit 1; }
|
||||
BUILD_ARGS="$BUILD_ARGS --build-arg $bs=$v"
|
||||
done
|
||||
# PROVENANCE: which commit are these bytes. Passed BOTH ways on
|
||||
# purpose. The --label sets it from outside for any Dockerfile; the
|
||||
# --build-arg feeds `ARG REVISION` for the Dockerfiles that stamp the
|
||||
# label themselves (hanzoai/cloud declares `ARG REVISION=unknown`,
|
||||
# and `unknown` is exactly what every image built without this said).
|
||||
#
|
||||
# An unlabelled image cannot be traced to source, and that stops being
|
||||
# cosmetic the moment two builders race for one tag: with one image
|
||||
# labelled and the other reading `unknown`, "which of these is the
|
||||
# release" had no answer short of diffing layers — while the wrong one
|
||||
# was already pinned. A version is a receipt only if the image can
|
||||
# name its own commit, so every image this fleet publishes now can.
|
||||
#
|
||||
# A Dockerfile with no `ARG REVISION` ignores the build-arg (buildkit
|
||||
# warns, never fails), so this is a no-op for every repo that has not
|
||||
# asked for it.
|
||||
PROV="--build-arg REVISION=$GITHUB_SHA"
|
||||
PROV="$PROV --label org.opencontainers.image.revision=$GITHUB_SHA"
|
||||
PROV="$PROV --label org.opencontainers.image.version=${ver:-$REL}"
|
||||
PROV="$PROV --label org.opencontainers.image.source=https://github.com/$GITHUB_REPOSITORY"
|
||||
# GIT_TOKEN (from KMS, via GITHUB_ENV) is passed as the `gh_token`
|
||||
# BuildKit secret so Dockerfiles can clone private Go modules; omitted
|
||||
# cleanly when absent (public-only builds unaffected).
|
||||
docker buildx build --platform "$plats" $BUILD_ARGS $PROV ${GIT_TOKEN:+--secret id=gh_token,env=GIT_TOKEN} --push $TAGS -f "$df" "$ctx"
|
||||
# The semver is the tag the fleet PINS, so prove it exists before the
|
||||
# run goes green — buildx can exit 0 while the manifest is not yet
|
||||
# resolvable, and a pin onto a phantom tag is an ImagePullBackOff.
|
||||
# The digest is printed with it because universe pins repo:tag@digest
|
||||
# and the kubelet honours the DIGEST: publishing both here is what
|
||||
# lets the two move together instead of being looked up separately
|
||||
# and drifting apart.
|
||||
pin="$repo:${ver:-$REL}${sfx:+-$sfx}"
|
||||
for i in 1 2 3 4 5 6; do
|
||||
dig=$(docker buildx imagetools inspect "$pin" --format '{{.Manifest.Digest}}' 2>/dev/null || true)
|
||||
[ -n "$dig" ] && break
|
||||
sleep 5
|
||||
done
|
||||
[ -n "$dig" ] || { echo "::error::$pin not resolvable after push"; exit 1; }
|
||||
echo "| \`$pin\` | \`$dig\` |" >> "$RUNNER_TEMP/pins.md"
|
||||
# Dual-host: mirror the exact tag set to registry.hanzo.ai (server-
|
||||
# side manifest copy — no rebuild). ghcr.io/<org>/<name> →
|
||||
# oci.hanzo.ai/<org>/<name>; public consumers keep ghcr, the fleet
|
||||
# is migrating to pull from ours. A skip here is now a WARNING, not
|
||||
# a notice: an image that never reaches our registry is the reason
|
||||
# a deploy still depends on GitHub, and that should be visible in
|
||||
# the run, not buried.
|
||||
if [ "${MIRROR_OK:-}" = "1" ]; then
|
||||
# crane, not buildx imagetools: the IAM token realm doesn't answer
|
||||
# buildx's multi-scope token request (spec gap, tracked).
|
||||
command -v crane >/dev/null 2>&1 || {
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
curl -fsSL https://github.com/google/go-containerregistry/releases/download/v0.20.2/go-containerregistry_Linux_x86_64.tar.gz \
|
||||
| tar -xz -C "$HOME/.local/bin" crane
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
}
|
||||
mrepo="oci.hanzo.ai/${repo#*/}"
|
||||
echo "$TAGS" | tr ' ' '\n' | grep -v '^-t$' | grep -v '^$' | while read -r ref; do
|
||||
crane copy "$ref" "${mrepo}:${ref##*:}" \
|
||||
|| echo "::warning::$ref did not reach oci.hanzo.ai (ghcr push unaffected)"
|
||||
done
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
done
|
||||
# The pins this run earned, ready to paste into universe. Tag and
|
||||
# digest are printed TOGETHER because they are one fact: the failure
|
||||
# this prevents is a new tag landing beside an old digest, which
|
||||
# reports the new version on the pod and serves the old bytes.
|
||||
if [ -s "$RUNNER_TEMP/pins.md" ]; then
|
||||
{ echo "### Pins"; echo; echo "| image | digest |"; echo "|---|---|";
|
||||
cat "$RUNNER_TEMP/pins.md"; } >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
|
||||
- name: Provision Go toolchain (go test gates on bare runners)
|
||||
# hanzo.yml `test:` gates (e.g. `go vet ./...`, `go test ...`) run
|
||||
# DIRECTLY on the runner, NOT inside a build container — but the stock
|
||||
@@ -1013,16 +839,72 @@ jobs:
|
||||
# unset) — so `go vet` dies with `could not read Username for github.com`.
|
||||
# Reuse the SAME token here for the runner's git (GIT_TOKEN, set by the
|
||||
# KMS step above; GH_PAT fallback). No-op when neither is present.
|
||||
#
|
||||
# AND OUR OWN MODULES RESOLVE FROM OUR OWN FORGE. A module path is a NAME,
|
||||
# not an address: `github.com/<owner>/x` is what the package is called, and
|
||||
# which host git dials for it is a separate decision. On a forge runner
|
||||
# that decision is the forge — the forge is where the code canonically
|
||||
# lives, and github.com is fed by a mirror that trails it, sometimes by
|
||||
# hours and sometimes not at all.
|
||||
#
|
||||
# Measured, on the first run in which this step's gate had ever actually
|
||||
# executed: hanzoai/cloud's go.mod carries `github.com/hanzoai/dashscopego
|
||||
# v0.6.0` as an indirect dependency. git.hanzo.ai serves it; github.com
|
||||
# answers `Repository not found`. So `go vet` died before a single test
|
||||
# ran, on a module that was never missing. Its own caller already knew the
|
||||
# answer — cloud's `containment` and `reach` jobs each make this exact
|
||||
# substitution, with a comment about the nine consecutive releases an
|
||||
# ACL drift on ONE sibling module once blocked — but the gate that runs
|
||||
# its tests reached the modules through a different door.
|
||||
#
|
||||
# go.sum is what makes this safe rather than merely convenient: the forge
|
||||
# serves the same objects, so the fetched zip hashes to the h1: line
|
||||
# already committed. A forge serving different bytes fails the build
|
||||
# loudly instead of shipping them.
|
||||
#
|
||||
# Longest prefix wins in git, so the owner-scoped forge rule outranks the
|
||||
# bare github.com rule beneath it and every OTHER module still resolves
|
||||
# from github.com. Both rules are optional and independent: no FORGE_TOKEN
|
||||
# (or a github.com-hosted run, where the server IS github.com) leaves the
|
||||
# old behaviour exactly as it was.
|
||||
#
|
||||
# SPLIT THE SERVER, do not strip a scheme you assumed. GITHUB_SERVER_URL
|
||||
# on this fleet's runners is `http://hanzo-git.hanzo.svc` — the forge's
|
||||
# in-cluster address, plain http — and a `${SERVER#https://}` that only
|
||||
# knows the https spelling passes it through whole, producing
|
||||
# `https://http://hanzo-git.hanzo.svc/...` and a git that resolves the host
|
||||
# `http`. Take the scheme and the host as the two values they are, and
|
||||
# compare the HOST against github.com so the guard cannot be fooled by a
|
||||
# spelling either.
|
||||
if: inputs.mode != 'delegate' && hashFiles('go.mod') != ''
|
||||
env:
|
||||
GH_PAT: ${{ secrets.GH_PAT }}
|
||||
FORGE_TOKEN: ${{ secrets.FORGE_TOKEN }}
|
||||
OWNER: ${{ github.repository_owner }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
TOKEN="${GIT_TOKEN:-${GH_PAT:-}}"
|
||||
if [ -z "$TOKEN" ]; then echo "no git token — public modules only"; exit 0; fi
|
||||
CFG="$RUNNER_TEMP/gitconfig-go-test"; : > "$CFG"
|
||||
GIT_CONFIG_GLOBAL="$CFG" git config --global \
|
||||
url."https://x-access-token:${TOKEN}@github.com/".insteadOf "https://github.com/"
|
||||
SERVER="${GITHUB_SERVER_URL:-https://github.com}"
|
||||
SCHEME="${SERVER%%://*}"; HOST="${SERVER#*://}"; HOST="${HOST%/}"
|
||||
wrote=0
|
||||
|
||||
if [ -n "${FORGE_TOKEN:-}" ] && [ "$HOST" != "github.com" ]; then
|
||||
GIT_CONFIG_GLOBAL="$CFG" git config --global \
|
||||
url."${SCHEME}://x:${FORGE_TOKEN}@${HOST}/${OWNER}/".insteadOf \
|
||||
"https://github.com/${OWNER}/"
|
||||
echo "${OWNER}/* modules resolve from ${SCHEME}://${HOST}"
|
||||
wrote=1
|
||||
fi
|
||||
|
||||
TOKEN="${GIT_TOKEN:-${GH_PAT:-}}"
|
||||
if [ -n "$TOKEN" ]; then
|
||||
GIT_CONFIG_GLOBAL="$CFG" git config --global \
|
||||
url."https://x-access-token:${TOKEN}@github.com/".insteadOf "https://github.com/"
|
||||
echo "github.com modules authenticated"
|
||||
wrote=1
|
||||
fi
|
||||
|
||||
[ "$wrote" = 1 ] || { echo "no git credential — public modules only"; exit 0; }
|
||||
{ echo "GIT_CONFIG_GLOBAL=$CFG"; echo "GIT_CONFIG_NOSYSTEM=1"; } >> "$GITHUB_ENV"
|
||||
echo "runner git authenticated for private Go modules"
|
||||
|
||||
@@ -1074,6 +956,227 @@ jobs:
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Build & push images (per hanzo.yml)
|
||||
# AFTER the gate, deliberately. This step used to run before `test:`,
|
||||
# so a repo whose gates FAILED still published an image: hanzoai/base
|
||||
# f9ec7067 pushed sha-f9ec706-amd64 on a run whose tests then went red.
|
||||
# Deploy and Publish site were skipped, so nothing shipped it — but the
|
||||
# tag existed, and a tag that exists is something a human pins by hand.
|
||||
# An artifact is a claim that the commit passed; it should not be
|
||||
# possible to produce one from a commit that did not.
|
||||
#
|
||||
# Nothing here needs the toolchains above it, and Deploy still follows
|
||||
# this step, so the only thing the move changes is that a red gate now
|
||||
# produces no image at all.
|
||||
if: inputs.mode != 'delegate'
|
||||
env:
|
||||
GH_PAT: ${{ secrets.GH_PAT }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# Test-only callers (hanzo.yml without `images:` — e.g. a repo whose
|
||||
# image lane lives in its own release.yml, or a pure library) skip the
|
||||
# build step entirely instead of exploding on a null .images.
|
||||
if [ "$(yq -r '.images // [] | length' hanzo.yml 2>/dev/null || echo 0)" = "0" ]; then
|
||||
echo "::notice::no images: in hanzo.yml — test-only caller, skipping build"; exit 0
|
||||
fi
|
||||
# Build-time private cross-org Go module read (the buildx `gh_token`
|
||||
# secret): prefer the KMS-fetched GIT_TOKEN, else fall back to the org
|
||||
# GH_PAT — the SAME BuildKit gh_token cloud's release.yml uses (proven
|
||||
# working). Keeps image builds green when the KMS deploy-cred fetch is
|
||||
# unavailable (a repo with a private cross-org dep like hanzoai/cloud
|
||||
# otherwise fails `go mod tidy` with git exit 128 in the buildx stage).
|
||||
# No-op for public-only builds when both are empty. Exported so the
|
||||
# `--secret id=gh_token,env=GIT_TOKEN` below reads it from the env.
|
||||
export GIT_TOKEN="${GIT_TOKEN:-${GH_PAT:-}}"
|
||||
SHORT=$(echo "${{ github.sha }}" | cut -c1-7)
|
||||
IS_TAG=$([ "${{ github.ref_type }}" = "tag" ] && echo 1 || echo 0)
|
||||
REL="${{ github.ref_name }}" # the git tag, verbatim: v1.26.19
|
||||
VER="${REL#v}" # v-stripped alias: 1.26.19
|
||||
yq -o=json -I=0 '.images' hanzo.yml | jq -c '.[]' | while read -r img; do
|
||||
name=$(echo "$img"|jq -r .name); ctx=$(echo "$img"|jq -r .context)
|
||||
df=$(echo "$img"|jq -r '.dockerfile // (.context+"/Dockerfile")'); repo=$(echo "$img"|jq -r .repo)
|
||||
# tag-suffix is OPTIONAL (most repos ship a single variant). When set
|
||||
# (e.g. "ce"/"ee") it qualifies every tag; when absent the tags are
|
||||
# clean (no trailing dash). Build + deploy must agree on this shape.
|
||||
sfx=$(echo "$img"|jq -r '."tag-suffix" // ""')
|
||||
# platforms is OPT-IN per image in hanzo.yml (default: amd64 only, so
|
||||
# every existing repo's tag shape "-amd64" is UNCHANGED). Set e.g.
|
||||
# platforms: [linux/amd64, linux/arm64]
|
||||
# to emit a multi-arch MANIFEST LIST — one digest serving both arches.
|
||||
# DOKS has no arm64 nodes, so arm64 builds via buildx QEMU emulation
|
||||
# (binfmt set up below); pure-Go (CGO_ENABLED=0) Dockerfiles that honor
|
||||
# $TARGETARCH cross-compile natively (fast, no emulation). For true
|
||||
# native-speed arm64, register a bare-metal arm64 host (spark/GB10) as
|
||||
# the hanzo-build-linux-arm64 self-hosted runner (values-build-arm64.yaml).
|
||||
plats=$(echo "$img"|jq -r '(.platforms // ["linux/amd64"]) | join(",")')
|
||||
# ---- Refuse a Go builder older than the module it compiles -------
|
||||
# The official golang images set GOTOOLCHAIN=local, so a go.mod
|
||||
# requiring a newer Go than the base image does not degrade — it
|
||||
# dies mid-build with "go.mod requires go >= X (running Y;
|
||||
# GOTOOLCHAIN=local)". hanzoai/visor v1.108.16 is the shipped
|
||||
# instance, and it is invisible before the image build: every local
|
||||
# `go build` succeeds because a dev box runs GOTOOLCHAIN=auto and
|
||||
# just downloads what the module asks for. So the break is
|
||||
# introduced by editing go.mod — a file with nothing to do with
|
||||
# Docker — and found by a red release build.
|
||||
#
|
||||
# It is not one repo's problem either: a sweep of every Dockerfile
|
||||
# across the orgs found 54 below their own go.mod in 23 repos, and
|
||||
# only 7 of 223 Go builder stages set GOTOOLCHAIN=auto. Fixing those
|
||||
# fixes today; this line is what makes the 55th impossible.
|
||||
#
|
||||
# Checked here, before any build work, so the run fails in seconds
|
||||
# rather than after binfmt, a version derivation and a layer cache.
|
||||
"$CI_HOME/bin/gover" "$df" "$ctx"
|
||||
# ---- The semver this build publishes ----------------------------
|
||||
# A branch build used to publish sha-<short7> and nothing else, so an
|
||||
# image only earned a version when a human remembered to cut a git
|
||||
# tag. That is the whole reason 14 of the fleet's 117 pins named a
|
||||
# commit instead of a release: not drift, just the only tag CI ever
|
||||
# offered them. Every build now publishes a version too, and the
|
||||
# sha- tag stays beside it for forensics.
|
||||
#
|
||||
# The number is DERIVED, never typed, by bin/imgver — the ONE
|
||||
# implementation, shared with the hand-rolled
|
||||
# .hanzo/workflows/deploy.yml that 11 repos carry instead of importing
|
||||
# this workflow (see .github/actions/imgver). Written twice it would be
|
||||
# right twice and then wrong once, which is exactly how `sha-<short>`
|
||||
# became the only tag those repos ever published. Read that script for
|
||||
# the rule; it is monotonic against the repo's manifest AND the
|
||||
# registry, so one tag can never cover two digests.
|
||||
ver=""
|
||||
if [ "$IS_TAG" = 1 ]; then
|
||||
# Release build: the git tag IS the version. Identity in, identity
|
||||
# out — deriving here would publish a number nobody asked for.
|
||||
:
|
||||
else
|
||||
# An explicit `version:` on the image entry wins over the repo-wide
|
||||
# one; both are optional, and imgver falls back to the manifest.
|
||||
iv=$(echo "$img"|jq -r '.version // ""')
|
||||
[ -z "$iv" ] && iv=$(yq -r '.version // ""' hanzo.yml)
|
||||
ver=$(IMGVER_VERSION="$iv" "$CI_HOME/bin/imgver" "$repo" "$ctx")
|
||||
fi
|
||||
if [ "$plats" = "linux/amd64" ]; then
|
||||
# single-arch: keep the exact legacy tag shape (-amd64) deploys expect.
|
||||
TAGS="-t $repo:sha-${SHORT}-amd64${sfx:+-$sfx} -t $repo:${sfx:+$sfx-}latest"
|
||||
else
|
||||
# multi-arch: one arch-neutral manifest-list tag (no -amd64 suffix).
|
||||
docker run --privileged --rm tonistiigi/binfmt --install arm64 >/dev/null 2>&1 || true
|
||||
TAGS="-t $repo:sha-${SHORT}${sfx:+-$sfx} -t $repo:${sfx:+$sfx-}latest"
|
||||
fi
|
||||
# Release (tag) build. The git tag IS the release name, so publish it
|
||||
# VERBATIM (v1.26.19) — that is the shape a universe CR pins, and
|
||||
# stripping the v is why releases were finished by hand-`crane copy`ing
|
||||
# sha-<sha7> onto the semver a human typed. Identity in, identity out.
|
||||
# The v-stripped alias stays for CRs already pinned that way (world
|
||||
# 2.4.51), and is skipped when a repo tags without a v. The old
|
||||
# `<ver>-amd64` alias is deleted: no CR in the fleet pinned it.
|
||||
if [ "$IS_TAG" = 1 ]; then
|
||||
TAGS="$TAGS -t $repo:${REL}${sfx:+-$sfx}"
|
||||
[ "$REL" != "$VER" ] && TAGS="$TAGS -t $repo:${VER}${sfx:+-$sfx}"
|
||||
else
|
||||
# The tag universe is meant to PIN. It goes on every build, not
|
||||
# only the ones somebody remembered to tag by hand.
|
||||
TAGS="$TAGS -t $repo:${ver}${sfx:+-$sfx}"
|
||||
fi
|
||||
echo "::group::build $name → $repo (${sfx}) [$plats]"
|
||||
# --build-arg assembly: static hanzo.yml `args` (a fixed value, e.g. a
|
||||
# pinned base image tag) + `build_secrets` (KMS values the KMS step
|
||||
# exported into the env above). Empty when a repo declares neither, so
|
||||
# the buildx line is unchanged for every existing repo.
|
||||
BUILD_ARGS=""
|
||||
while IFS= read -r kv; do [ -n "$kv" ] && BUILD_ARGS="$BUILD_ARGS --build-arg $kv"; done \
|
||||
< <(echo "$img" | jq -r '(.args // {}) | to_entries[] | "\(.key)=\(.value)"')
|
||||
for bs in $(echo "$img" | jq -r '(.build_secrets // [])[]'); do
|
||||
v=$(printenv "$bs" 2>/dev/null || true)
|
||||
# Defense in depth: the KMS step above already refuses to continue
|
||||
# with an unresolved build_secret, so reaching here empty means the
|
||||
# two steps disagree. Stop rather than bake a blank credential.
|
||||
[ -z "$v" ] && { echo "::error::build_secret $bs is unset at build time — refusing to bake an empty value"; exit 1; }
|
||||
BUILD_ARGS="$BUILD_ARGS --build-arg $bs=$v"
|
||||
done
|
||||
# PROVENANCE: which commit are these bytes. Passed BOTH ways on
|
||||
# purpose. The --label sets it from outside for any Dockerfile; the
|
||||
# --build-arg feeds `ARG REVISION` for the Dockerfiles that stamp the
|
||||
# label themselves (hanzoai/cloud declares `ARG REVISION=unknown`,
|
||||
# and `unknown` is exactly what every image built without this said).
|
||||
#
|
||||
# An unlabelled image cannot be traced to source, and that stops being
|
||||
# cosmetic the moment two builders race for one tag: with one image
|
||||
# labelled and the other reading `unknown`, "which of these is the
|
||||
# release" had no answer short of diffing layers — while the wrong one
|
||||
# was already pinned. A version is a receipt only if the image can
|
||||
# name its own commit, so every image this fleet publishes now can.
|
||||
#
|
||||
# A Dockerfile with no `ARG REVISION` ignores the build-arg (buildkit
|
||||
# warns, never fails), so this is a no-op for every repo that has not
|
||||
# asked for it.
|
||||
PROV="--build-arg REVISION=$GITHUB_SHA"
|
||||
PROV="$PROV --label org.opencontainers.image.revision=$GITHUB_SHA"
|
||||
PROV="$PROV --label org.opencontainers.image.version=${ver:-$REL}"
|
||||
PROV="$PROV --label org.opencontainers.image.source=https://github.com/$GITHUB_REPOSITORY"
|
||||
# GIT_TOKEN (from KMS, via GITHUB_ENV) is passed as the `gh_token`
|
||||
# BuildKit secret so Dockerfiles can clone private Go modules; omitted
|
||||
# cleanly when absent (public-only builds unaffected).
|
||||
docker buildx build --platform "$plats" $BUILD_ARGS $PROV ${GIT_TOKEN:+--secret id=gh_token,env=GIT_TOKEN} --push $TAGS -f "$df" "$ctx"
|
||||
# The semver is the tag the fleet PINS, so prove it exists before the
|
||||
# run goes green — buildx can exit 0 while the manifest is not yet
|
||||
# resolvable, and a pin onto a phantom tag is an ImagePullBackOff.
|
||||
# The digest is printed with it because universe pins repo:tag@digest
|
||||
# and the kubelet honours the DIGEST: publishing both here is what
|
||||
# lets the two move together instead of being looked up separately
|
||||
# and drifting apart.
|
||||
pin="$repo:${ver:-$REL}${sfx:+-$sfx}"
|
||||
for i in 1 2 3 4 5 6; do
|
||||
dig=$(docker buildx imagetools inspect "$pin" --format '{{.Manifest.Digest}}' 2>/dev/null || true)
|
||||
[ -n "$dig" ] && break
|
||||
sleep 5
|
||||
done
|
||||
[ -n "$dig" ] || { echo "::error::$pin not resolvable after push"; exit 1; }
|
||||
echo "| \`$pin\` | \`$dig\` |" >> "$RUNNER_TEMP/pins.md"
|
||||
# Dual-host: mirror the exact tag set to registry.hanzo.ai (server-
|
||||
# side manifest copy — no rebuild). ghcr.io/<org>/<name> →
|
||||
# registry.hanzo.ai/<org>/<name>; public consumers keep ghcr, the fleet
|
||||
# is migrating to pull from ours. A skip here is now a WARNING, not
|
||||
# a notice: an image that never reaches our registry is the reason
|
||||
# a deploy still depends on GitHub, and that should be visible in
|
||||
# the run, not buried.
|
||||
if [ "${MIRROR_OK:-}" = "1" ]; then
|
||||
# crane, not buildx imagetools: the IAM token realm doesn't answer
|
||||
# buildx's multi-scope token request (spec gap, tracked).
|
||||
command -v crane >/dev/null 2>&1 || {
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
curl -fsSL https://github.com/google/go-containerregistry/releases/download/v0.20.2/go-containerregistry_Linux_x86_64.tar.gz \
|
||||
| tar -xz -C "$HOME/.local/bin" crane
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
}
|
||||
# ONE mapping, stated once. The GHCR org and the registry org are
|
||||
# not the same word — ghcr.io/hanzoai/x mirrors to
|
||||
# registry.hanzo.ai/hanzo/x — and a second copy of this table is
|
||||
# how a push path and a pin drift into naming different images.
|
||||
rorg="${repo#*/}"; rname="${rorg#*/}"; rorg="${rorg%%/*}"
|
||||
case "$rorg" in
|
||||
hanzoai) rorg=hanzo ;;
|
||||
zooai) rorg=zoo ;;
|
||||
*) ;; # bootnode, lux-private, ... map to themselves
|
||||
esac
|
||||
mrepo="registry.hanzo.ai/$rorg/$rname"
|
||||
echo "$TAGS" | tr ' ' '\n' | grep -v '^-t$' | grep -v '^$' | while read -r ref; do
|
||||
crane copy "$ref" "${mrepo}:${ref##*:}" \
|
||||
|| echo "::warning::$ref did not reach registry.hanzo.ai (ghcr push unaffected)"
|
||||
done
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
done
|
||||
# The pins this run earned, ready to paste into universe. Tag and
|
||||
# digest are printed TOGETHER because they are one fact: the failure
|
||||
# this prevents is a new tag landing beside an old digest, which
|
||||
# reports the new version on the pod and serves the old bytes.
|
||||
if [ -s "$RUNNER_TEMP/pins.md" ]; then
|
||||
{ echo "### Pins"; echo; echo "| image | digest |"; echo "|---|---|";
|
||||
cat "$RUNNER_TEMP/pins.md"; } >> "$GITHUB_STEP_SUMMARY"
|
||||
fi
|
||||
|
||||
- name: Client — commit the projection and cut the patch
|
||||
# LAST, and after `test:` on purpose: this pushes a tag, the tag starts a
|
||||
# publish, and a published client is the one artifact in this workflow
|
||||
|
||||
+45
-16
@@ -38,19 +38,31 @@
|
||||
# Only the two LABELLED markers git actually writes:
|
||||
#
|
||||
# ^<<<<<<< <label> the start marker, seven '<' then a space
|
||||
# ^======= the divider, seven bare equals
|
||||
# ^>>>>>>> <label> the end marker, seven '>' then a space
|
||||
#
|
||||
# The middle `=======` divider is deliberately NOT matched. It is seven bare
|
||||
# equals signs at the start of a line, which is also how reStructuredText and
|
||||
# Setext-style Markdown underline a heading — a real and common shape in our
|
||||
# docs. Matching it would fail repos that have nothing wrong with them, and a
|
||||
# gate that fails what ought to pass is a gate that gets switched off. The two
|
||||
# angle-bracket forms carry a label and have no such collision.
|
||||
# ALL THREE, IN THAT ORDER, EACH ON ITS OWN LINE. Not any one of them — the
|
||||
# whole shape. That is not belt-and-braces, it is the difference between a gate
|
||||
# and a nuisance, and it was measured the expensive way: matching the start
|
||||
# marker ALONE failed hanzoai/app, whose builder tests carry
|
||||
#
|
||||
# Exactly seven, anchored at column 1, followed by a space: `<<<<<<<<` (eight,
|
||||
# a heredoc or a C++ stream) does not match, and neither does an indented
|
||||
# mention inside a comment explaining conflict markers — which is what lets this
|
||||
# file describe them without flagging itself.
|
||||
# <<<<<<< START_TITLE index.html >>>>>>> END_TITLE
|
||||
#
|
||||
# as literal fixture data. That line opens with seven '<' and a space and is
|
||||
# not a merge conflict — both markers sit on ONE line, which is a thing git
|
||||
# never writes. A start marker that also carries an end marker is therefore
|
||||
# excluded outright, and a file must show the divider and a separate end line
|
||||
# before this refuses it.
|
||||
#
|
||||
# The divider is only ever matched as part of that conjunction, never alone:
|
||||
# seven bare equals signs at column 1 is also how reStructuredText and Setext
|
||||
# Markdown underline a heading, and failing every doc in the estate is how a
|
||||
# gate gets switched off.
|
||||
#
|
||||
# Exactly seven, anchored at column 1: `<<<<<<<<` (eight — a heredoc, a C++
|
||||
# stream) does not match, and neither does an indented mention inside a comment
|
||||
# explaining conflict markers, which is what lets this file describe them
|
||||
# without flagging itself.
|
||||
#
|
||||
# Tracked files only, via `git grep`, so a stray marker in an untracked scratch
|
||||
# file or in node_modules cannot fail a build. `-I` skips binaries.
|
||||
@@ -61,18 +73,35 @@ cd "${1:-.}"
|
||||
git rev-parse --is-inside-work-tree >/dev/null 2>&1 || {
|
||||
echo "conflictmarkers: not a git work tree ($(pwd)) — skipping"; exit 0; }
|
||||
|
||||
# Built from a variable rather than written literally, so this script is not its
|
||||
# Built from variables rather than written literally, so this script is not its
|
||||
# own first finding when it scans the repo that carries it.
|
||||
L=$(printf '<%.0s' $(seq 7))
|
||||
E=$(printf '=%.0s' $(seq 7))
|
||||
R=$(printf '>%.0s' $(seq 7))
|
||||
|
||||
hits=$(git grep -I -n -E "^(${L}|${R}) " -- . || true)
|
||||
# Candidate files: those carrying a start marker that is NOT also an end marker
|
||||
# on the same line. `-l` for names only; the line numbers come later, per file.
|
||||
cands=$(git grep -I -l -E "^${L} " -- . || true)
|
||||
|
||||
if [ -n "$hits" ]; then
|
||||
bad=""
|
||||
for f in $cands; do
|
||||
grep -qE "^${L} " -- "$f" 2>/dev/null || continue
|
||||
# The same-line form (app's fixture) is not a conflict. Require at least one
|
||||
# start line that does not also carry an end marker.
|
||||
grep -E "^${L} " -- "$f" 2>/dev/null | grep -qv -- "$R" || continue
|
||||
# And require the rest of the shape: a bare divider and a separate end line.
|
||||
grep -qE "^${E}$" -- "$f" 2>/dev/null || continue
|
||||
grep -E "^${R} " -- "$f" 2>/dev/null | grep -qv -- "$L" || continue
|
||||
bad="$bad $f"
|
||||
done
|
||||
|
||||
if [ -n "$bad" ]; then
|
||||
echo "::error::committed merge-conflict markers — an unresolved merge is in the tree"
|
||||
echo "$hits" | while IFS= read -r line; do
|
||||
echo " $line"
|
||||
echo "::error file=${line%%:*}::committed conflict marker"
|
||||
for f in $bad; do
|
||||
grep -nE "^(${L} |${E}$|${R} )" -- "$f" | while IFS= read -r line; do
|
||||
echo " $f:$line"
|
||||
done
|
||||
echo "::error file=${f}::committed conflict marker"
|
||||
done
|
||||
echo
|
||||
echo "Resolve the merge and commit the result. If the file is GENERATED, do not"
|
||||
|
||||
Executable
+238
@@ -0,0 +1,238 @@
|
||||
#!/usr/bin/env bash
|
||||
# sitedeploy — publish a built static export to the Hanzo PaaS Sites plane.
|
||||
# One implementation, every static site.
|
||||
#
|
||||
# sitedeploy <slug> <dir> # e.g. sitedeploy hanzo-ai out
|
||||
#
|
||||
# WHY THIS IS A SCRIPT AND NOT INLINE SHELL: the same forty lines of enqueue →
|
||||
# upload → complete were about to be pasted into hanzo.ai, hanzo.app, hips,
|
||||
# computer and every static surface after them. Written N times it is right N-1
|
||||
# times and then wrong once — which is exactly how `sha-<short>` became the only
|
||||
# tag eleven repos ever published (see bin/imgver). The plane's contract lives
|
||||
# here, once.
|
||||
#
|
||||
# THE THREE STEPS, and why the bytes never pass through the API:
|
||||
#
|
||||
# POST /v1/projects/<slug>/deploy -> 202 {id, bucket, prefix, upload}
|
||||
# POST <upload.url> per file -> the bytes, straight to S3
|
||||
# POST /v1/projects/<slug>/deployments/<id>/complete
|
||||
#
|
||||
# A real export is large — hanzo.ai is 128 MB across 8403 files — and cloud's
|
||||
# BodyLimit is 16 MiB, so an artifact POST is refused by fasthttp BEFORE any
|
||||
# handler runs, and it fails as an opaque 400 "Error when parsing request" that
|
||||
# reads like a malformed payload rather than a size cap. The git source is the
|
||||
# documented route for exactly this shape.
|
||||
#
|
||||
# NO STANDING S3 CREDENTIAL. The 202 carries `upload`: a presigned POST policy
|
||||
# that is prefix-scoped (starts-with $key "<org>/<slug>/", enforced by S3 itself),
|
||||
# short-lived (30 min) and size-bounded. That replaced handing every repo the
|
||||
# bucket's own long-lived access key — one key for a bucket whose only tenant
|
||||
# separation is the key prefix, so every repo holding it could overwrite EVERY
|
||||
# org's site (cloud apps/projects/grant.go). Do not reintroduce SITES_S3_* here.
|
||||
# HANZO_DEPLOY_TOKEN is the ONE credential this needs.
|
||||
#
|
||||
# DELETION IS THE SERVER'S. The grant authorizes writes only, so CI cannot remove
|
||||
# a file; `keys` in the completion is the manifest cloud reconciles the prefix
|
||||
# against, and it deletes what the build no longer produces. That is where
|
||||
# `aws s3 sync --delete` went. It fails CLOSED on an empty manifest, and so does
|
||||
# this script — a build that enumerated nothing has failed, and honouring it
|
||||
# literally would delete the live site.
|
||||
#
|
||||
# ENV: HANZO_DEPLOY_TOKEN (required) the org-scoped `sk-` key; mint at POST /v1/keys
|
||||
# HANZO_API (https://api.hanzo.ai)
|
||||
# SITEDEPLOY_JOBS (24) parallel uploads
|
||||
# SITEDEPLOY_COMMIT / SITEDEPLOY_BRANCH recorded on the deployment
|
||||
# SITEDEPLOY_PLAN=1 print the manifest and exit; no network. The test seam.
|
||||
set -euo pipefail
|
||||
|
||||
slug="${1:?usage: sitedeploy <slug> <dir>}"
|
||||
dir="${2:?usage: sitedeploy <slug> <dir>}"
|
||||
api="${HANZO_API:-https://api.hanzo.ai}"
|
||||
jobs="${SITEDEPLOY_JOBS:-24}"
|
||||
|
||||
[ -d "$dir" ] || { echo "::error::$dir is not a directory — the build produced no export"; exit 1; }
|
||||
|
||||
# ---- content type -----------------------------------------------------------
|
||||
# The presigned POST carries no Content-Type condition, so whatever CI sends is
|
||||
# what the object stores — and what it stores is what the edge serves. Send
|
||||
# nothing and every page is application/octet-stream, which a browser DOWNLOADS
|
||||
# instead of rendering: a green deploy that serves an unusable site.
|
||||
ctype() {
|
||||
case "${1##*.}" in
|
||||
html|htm) echo 'text/html; charset=utf-8' ;;
|
||||
css) echo 'text/css; charset=utf-8' ;;
|
||||
js|mjs) echo 'text/javascript; charset=utf-8' ;;
|
||||
json) echo 'application/json; charset=utf-8' ;;
|
||||
xml) echo 'application/xml; charset=utf-8' ;;
|
||||
txt) echo 'text/plain; charset=utf-8' ;;
|
||||
svg) echo 'image/svg+xml' ;;
|
||||
png) echo 'image/png' ;;
|
||||
jpg|jpeg) echo 'image/jpeg' ;;
|
||||
gif) echo 'image/gif' ;;
|
||||
webp) echo 'image/webp' ;;
|
||||
avif) echo 'image/avif' ;;
|
||||
ico) echo 'image/x-icon' ;;
|
||||
woff) echo 'font/woff' ;;
|
||||
woff2) echo 'font/woff2' ;;
|
||||
ttf) echo 'font/ttf' ;;
|
||||
otf) echo 'font/otf' ;;
|
||||
wasm) echo 'application/wasm' ;;
|
||||
pdf) echo 'application/pdf' ;;
|
||||
webmanifest) echo 'application/manifest+json' ;;
|
||||
map) echo 'application/json' ;;
|
||||
*) echo 'application/octet-stream' ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ---- cache control ----------------------------------------------------------
|
||||
# Mirrors cloud's apps/sites.CacheControlFor so a site cached the same whichever
|
||||
# route its bytes took. bin/sitedeploy_test.sh pins these strings; if the server
|
||||
# changes its policy the test is what catches the drift.
|
||||
#
|
||||
# A fingerprinted asset (Vite/Next/webpack emit app.4f3a9c21.js) is immutable: a
|
||||
# new build changes the hash, so the old URL can be cached forever.
|
||||
#
|
||||
# The class is written `[._-]` with the dash LAST and the pattern held in a
|
||||
# variable. Go's regexp spells the same set `[.\-_]`, and transcribing that
|
||||
# literally into `[[ =~ ]]` is a silent defect twice over: a backslash is literal
|
||||
# inside a POSIX bracket expression, so `\-_` reads as the RANGE \…_ and the shell
|
||||
# rejects it as "invalid character range" — and because the `if` merely evaluates
|
||||
# false, every fingerprinted asset quietly fell back to max-age=3600 instead of
|
||||
# immutable. An unquoted inline pattern also has its own quoting hazards; the
|
||||
# variable form is the one that is read as a regex rather than a glob.
|
||||
fingerprint_re='[._-][0-9a-fA-F]{8,}\.[a-z0-9]+$'
|
||||
cachectl() {
|
||||
local k="$1" base="${1##*/}"
|
||||
case "${k##*.}" in
|
||||
html|htm) echo 'public, max-age=60, s-maxage=86400' ;;
|
||||
js|mjs|css|woff|woff2|png|jpg|jpeg|gif|svg|webp|avif|ico|ttf|otf|wasm|data|pck|unityweb|mem)
|
||||
if [[ "$base" =~ $fingerprint_re ]]; then
|
||||
echo 'public, max-age=31536000, immutable'
|
||||
else
|
||||
echo 'public, max-age=3600'
|
||||
fi ;;
|
||||
*) echo 'public, max-age=3600' ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ---- the manifest -----------------------------------------------------------
|
||||
# Paths RELATIVE to the export root, which is exactly what the completion's
|
||||
# `keys` must carry (cloud reconciles `keep[rel]` against them).
|
||||
#
|
||||
# CNAME does not travel. It is a GitHub Pages artifact that means nothing to S3
|
||||
# and would ship a stale hostname claim into the bucket.
|
||||
manifest=$(cd "$dir" && find . -type f ! -name CNAME | sed 's|^\./||' | LC_ALL=C sort)
|
||||
count=$(printf '%s' "$manifest" | grep -c . || true)
|
||||
[ "$count" -gt 0 ] || { echo "::error::$dir contains no files — refusing to deploy an empty manifest"; exit 1; }
|
||||
|
||||
if [ -n "${SITEDEPLOY_PLAN:-}" ]; then
|
||||
printf 'slug=%s dir=%s files=%s\n' "$slug" "$dir" "$count"
|
||||
while IFS= read -r k; do [ -n "$k" ] && printf '%s\t%s\t%s\n' "$k" "$(ctype "$k")" "$(cachectl "$k")"; done <<< "$manifest"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
: "${HANZO_DEPLOY_TOKEN:?HANZO_DEPLOY_TOKEN is unset. It is the ONE credential this needs; set it on the forge (git.hanzo.ai), which is what reads .hanzo/workflows — not on GitHub. Mint: POST /v1/keys {\"type\":\"secret\"}}"
|
||||
|
||||
# ---- 0. ensure the project exists (self-provision) --------------------------
|
||||
# A brand-new slug has no project, and the enqueue below 404s on a missing one —
|
||||
# which is why every never-before-deployed site failed its first run. Create it
|
||||
# idempotently here instead: 200/201 the first time, 409 every time after, both
|
||||
# success. So a new site needs no out-of-band `POST /v1/projects`; committing the
|
||||
# workflow is enough. The repo is linked because the git-source enqueue requires
|
||||
# a linked project (deployGit rejects an unlinked one); SITEDEPLOY_REPO overrides
|
||||
# the origin for a forge checkout whose remote is not the canonical repo.
|
||||
repo="${SITEDEPLOY_REPO:-$(git remote get-url origin 2>/dev/null || true)}"
|
||||
pcode=$(curl -sS -o /tmp/sd-proj.json -w '%{http_code}' \
|
||||
-X POST "$api/v1/projects" \
|
||||
-H "Authorization: Bearer $HANZO_DEPLOY_TOKEN" -H 'Content-Type: application/json' \
|
||||
-d "$(jq -nc --arg s "$slug" --arg u "$repo" --arg b "${SITEDEPLOY_BRANCH:-main}" \
|
||||
'{slug:$s, name:$s} + (if $u=="" then {} else {repo:{url:$u, branch:$b}} end)')")
|
||||
case "$pcode" in
|
||||
200|201|409) : ;; # created now, or already there — either is the state we need
|
||||
*) echo "::error::ensure project $slug returned HTTP $pcode"; head -c 400 /tmp/sd-proj.json; echo; exit 1 ;;
|
||||
esac
|
||||
|
||||
# ---- 1. enqueue -------------------------------------------------------------
|
||||
# 202 Accepted is the success code: the deployment is queued, not live. bucket
|
||||
# and prefix come FROM cloud (sitePrefix(org, slug) is server-side) — never guess
|
||||
# them, or the upload lands where nothing is served the moment an org or slug
|
||||
# changes.
|
||||
code=$(curl -sS -o /tmp/sd-enq.json -w '%{http_code}' \
|
||||
-X POST "$api/v1/projects/$slug/deploy" \
|
||||
-H "Authorization: Bearer $HANZO_DEPLOY_TOKEN" -H 'Content-Type: application/json' \
|
||||
-d "{\"source\":\"git\",\"commit\":\"${SITEDEPLOY_COMMIT:-}\",\"branch\":\"${SITEDEPLOY_BRANCH:-main}\"}")
|
||||
if [ "$code" != "202" ]; then
|
||||
echo "::error::enqueue $api/v1/projects/$slug/deploy returned HTTP $code"; head -c 600 /tmp/sd-enq.json; echo; exit 1
|
||||
fi
|
||||
dep=$(jq -r '.id' /tmp/sd-enq.json)
|
||||
prefix=$(jq -r '.prefix' /tmp/sd-enq.json)
|
||||
upload_url=$(jq -r '.upload.url // empty' /tmp/sd-enq.json)
|
||||
if [ -z "$upload_url" ]; then
|
||||
echo "::error::the 202 carried no upload grant, so there is no way to write the bytes."
|
||||
echo " cloud mints one only when presigning is configured (S3_ADMIN_* on the cloud deployment)."
|
||||
exit 1
|
||||
fi
|
||||
jq -c '{id,version,status,bucket,prefix}' /tmp/sd-enq.json
|
||||
|
||||
# A build that dies after this point would leave the deployment "queued" and the
|
||||
# project stuck "building" forever. Report the failure so cloud records an honest
|
||||
# terminal state instead of a lie by omission.
|
||||
fail() {
|
||||
curl -sS -X POST "$api/v1/projects/$slug/deployments/$dep/complete" \
|
||||
-H "Authorization: Bearer $HANZO_DEPLOY_TOKEN" -H 'Content-Type: application/json' \
|
||||
-d '{"status":"error","message":"CI upload failed"}' >/dev/null 2>&1 || true
|
||||
echo "::error::marked deployment $dep as error"
|
||||
}
|
||||
trap 'fail' ERR
|
||||
|
||||
# ---- 2. the bytes -----------------------------------------------------------
|
||||
# `file` goes LAST: S3 ignores every field after the file part, so a grant field
|
||||
# trailing the body is silently dropped and the signature check fails.
|
||||
#
|
||||
# `key` is DROPPED from the grant's fields and re-sent per object. The grant
|
||||
# carries key="<org>/<slug>/" — the starts-with PLACEHOLDER, not a destination —
|
||||
# and forwarding it verbatim alongside the real key posts `key` twice, which S3
|
||||
# answers 400 for every object. That is the whole of the first end-to-end run:
|
||||
# 8403 files, 8403 400s. Everything else in the map (bucket, policy, x-amz-*) is
|
||||
# covered by the signature and must travel untouched.
|
||||
jq -r '.upload.fields | to_entries[] | select(.key != "key") | "-F\n\(.key)=\(.value)"' /tmp/sd-enq.json > /tmp/sd-fields
|
||||
put() {
|
||||
local rel="$1" args=() line
|
||||
# A read loop, NOT `mapfile`: mapfile is bash 4+, and macOS ships bash 3.2, so
|
||||
# on a dev box it fails as `command not found`, the array stays EMPTY, and every
|
||||
# upload goes out with no policy or signature at all. S3 then rejects it for a
|
||||
# malformed X-Amz-Credential — an error that points at the credential rather
|
||||
# than at the array that never got built.
|
||||
while IFS= read -r line; do args+=("$line"); done < /tmp/sd-fields
|
||||
curl -sS --fail-with-body -o /dev/null \
|
||||
-X POST "$UP_URL" \
|
||||
-F "key=$PREFIX/$rel" "${args[@]}" \
|
||||
-F "Content-Type=$(ctype "$rel")" -F "Cache-Control=$(cachectl "$rel")" \
|
||||
-F "file=@$DIR/$rel" \
|
||||
|| { echo "::error::upload failed: $rel"; return 1; }
|
||||
}
|
||||
export -f put ctype cachectl
|
||||
export UP_URL="$upload_url" PREFIX="$prefix" DIR="$dir" fingerprint_re
|
||||
|
||||
echo "uploading $count files to s3://$(jq -r .bucket /tmp/sd-enq.json)/$prefix ($jobs parallel)"
|
||||
printf '%s\n' "$manifest" | grep . | xargs -P "$jobs" -I{} bash -c 'put "$@"' _ {}
|
||||
|
||||
# ---- 3. flip it live --------------------------------------------------------
|
||||
# `wc -c`, not `stat`: the size flag is spelled -f%z on BSD and -c%s on GNU, and a
|
||||
# `stat -f%z || stat -c%s` fallback does not work — on Linux the first arm fails
|
||||
# INSIDE the pipeline, awk still exits 0, and the `||` never fires, so the byte
|
||||
# count silently reports 0. wc is the one spelling both agree on.
|
||||
bytes=$(cd "$dir" && find . -type f ! -name CNAME -exec wc -c {} + | awk '$2!="total"{s+=$1} END{print s+0}')
|
||||
jq -n --arg c "${SITEDEPLOY_COMMIT:-}" --argjson f "$count" --argjson b "${bytes:-0}" \
|
||||
--args '{status:"live",commit:$c,files:$f,bytes:$b,keys:$ARGS.positional}' \
|
||||
$(printf '%s\n' "$manifest" | grep .) > /tmp/sd-done.json
|
||||
|
||||
code=$(curl -sS -o /tmp/sd-resp.json -w '%{http_code}' \
|
||||
-X POST "$api/v1/projects/$slug/deployments/$dep/complete" \
|
||||
-H "Authorization: Bearer $HANZO_DEPLOY_TOKEN" -H 'Content-Type: application/json' \
|
||||
--data-binary @/tmp/sd-done.json)
|
||||
trap - ERR
|
||||
if [ "$code" != "200" ]; then
|
||||
echo "::error::complete returned HTTP $code"; head -c 600 /tmp/sd-resp.json; echo; fail; exit 1
|
||||
fi
|
||||
jq -c '{status,liveUrl,version,files,bytes}' /tmp/sd-resp.json
|
||||
Executable
+80
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tests for bin/sitedeploy. Runs OFFLINE: SITEDEPLOY_PLAN=1 stops the script
|
||||
# before the first network call and prints the manifest it would upload, so every
|
||||
# case here is deterministic and needs no token, no bucket and no cluster.
|
||||
# Run: bash bin/sitedeploy_test.sh
|
||||
set -uo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
SD="$PWD/bin/sitedeploy"
|
||||
tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT
|
||||
fail=0
|
||||
|
||||
plan() { SITEDEPLOY_PLAN=1 bash "$SD" a-slug "$1" 2>&1; }
|
||||
t() { # t <name> <got> <want>
|
||||
if [ "$2" = "$3" ]; then printf 'ok %-56s -> %s\n' "$1" "$2"
|
||||
else printf 'FAIL %-56s -> %s (want %s)\n' "$1" "$2" "$3"; fail=1; fi
|
||||
}
|
||||
# field <dir> <key> <col> — the ctype (2) or cachectl (3) column for one key
|
||||
field() { plan "$1" | awk -F'\t' -v k="$2" -v c="$3" '$1==k{print $c}'; }
|
||||
|
||||
site="$tmp/site"; mkdir -p "$site/assets" "$site/nested/deep"
|
||||
echo '<h1>hi</h1>' > "$site/index.html"
|
||||
echo 'body{}' > "$site/assets/app.4f3a9c21.css"
|
||||
echo 'x' > "$site/assets/plain.css"
|
||||
echo 'y' > "$site/assets/chunk-AB12CD34.js"
|
||||
echo '{}' > "$site/data.json"
|
||||
echo 'z' > "$site/nested/deep/page.html"
|
||||
echo 'hanzo.ai' > "$site/CNAME"
|
||||
|
||||
# --- the manifest ------------------------------------------------------------
|
||||
# Keys are RELATIVE to the export root: cloud reconciles keep[rel] against them,
|
||||
# so a leading ./ or an absolute path would match nothing and the completion
|
||||
# would prune the entire live site.
|
||||
t "keys are relative, no leading ./" "$(plan "$site" | awk -F'\t' 'NR>1&&$1~/^\.?\//{print "ABS"}' | head -1)" ""
|
||||
t "nested paths keep their subdirs" "$(plan "$site" | awk -F'\t' '$1=="nested/deep/page.html"{print "yes"}')" "yes"
|
||||
# CNAME is a GitHub Pages artifact: it means nothing to S3 and would ship a stale
|
||||
# hostname claim into the bucket.
|
||||
t "CNAME does not travel" "$(plan "$site" | awk -F'\t' '$1=="CNAME"{print "leaked"}')" ""
|
||||
t "file count excludes CNAME" "$(plan "$site" | head -1 | grep -o 'files=[0-9]*')" "files=6"
|
||||
|
||||
# --- content type ------------------------------------------------------------
|
||||
# The presigned POST carries no Content-Type condition, so what CI sends is what
|
||||
# the object stores and what the edge serves. Send nothing and a browser
|
||||
# DOWNLOADS every page instead of rendering it.
|
||||
t "html" "$(field "$site" index.html 2)" "text/html; charset=utf-8"
|
||||
t "css" "$(field "$site" assets/plain.css 2)" "text/css; charset=utf-8"
|
||||
t "js" "$(field "$site" assets/chunk-AB12CD34.js 2)" "text/javascript; charset=utf-8"
|
||||
t "json" "$(field "$site" data.json 2)" "application/json; charset=utf-8"
|
||||
|
||||
# --- cache control: mirrors cloud apps/sites.CacheControlFor -----------------
|
||||
# These strings are the SERVER's policy, pinned here so the two cannot drift
|
||||
# apart silently. If cloud changes CacheControlFor, this is what goes red.
|
||||
t "html is short-lived, long at the edge" "$(field "$site" index.html 3)" "public, max-age=60, s-maxage=86400"
|
||||
t "unfingerprinted asset is an hour" "$(field "$site" assets/plain.css 3)" "public, max-age=3600"
|
||||
# The regression this pins: Go spells the class [.\-_], and transcribing that
|
||||
# into [[ =~ ]] makes the shell reject it as an invalid character range. The `if`
|
||||
# then merely evaluates false, so every hashed asset silently lost `immutable`.
|
||||
t "fingerprinted .hash. is immutable" "$(field "$site" assets/app.4f3a9c21.css 3)" "public, max-age=31536000, immutable"
|
||||
t "fingerprinted -HASH- is immutable" "$(field "$site" assets/chunk-AB12CD34.js 3)" "public, max-age=31536000, immutable"
|
||||
|
||||
# --- fail closed -------------------------------------------------------------
|
||||
# reconcilePrefix deletes whatever the manifest omits, so an empty manifest is a
|
||||
# request to delete the live site. A build that enumerated nothing has failed.
|
||||
empty="$tmp/empty"; mkdir -p "$empty"
|
||||
plan "$empty" >/dev/null 2>&1
|
||||
t "empty export is refused" "$?" "1"
|
||||
only_cname="$tmp/onlycname"; mkdir -p "$only_cname"; echo x > "$only_cname/CNAME"
|
||||
plan "$only_cname" >/dev/null 2>&1
|
||||
t "a dir holding only CNAME is empty too" "$?" "1"
|
||||
plan "$tmp/does-not-exist" >/dev/null 2>&1
|
||||
t "missing export dir is refused" "$?" "1"
|
||||
|
||||
# --- the credential ----------------------------------------------------------
|
||||
# Not in PLAN mode (that is the offline seam), but a real run must refuse to
|
||||
# start rather than enqueue a deployment it cannot complete.
|
||||
out=$(HANZO_DEPLOY_TOKEN= bash "$SD" a-slug "$site" 2>&1); rc=$?
|
||||
t "no token: exits non-zero" "$rc" "1"
|
||||
t "no token: says which secret" "$(printf '%s' "$out" | grep -c HANZO_DEPLOY_TOKEN)" "1"
|
||||
|
||||
[ $fail -eq 0 ] && echo "PASS" || echo "FAIL"
|
||||
exit $fail
|
||||
Reference in New Issue
Block a user