cli/.hanzo/workflows/release-matrix.yml

name: Release Matrix (CLI)

# THIS FILE RUNS ON THE FORGE (git.hanzo.ai), and that is why it lives in
# `.hanzo/workflows/` rather than `.github/workflows/`.
#
# `runs-on: hanzo-build-linux-amd64` was always a FORGE label. It is one of 28
# advertised by the git-runner pool (hanzo/git-runner-0..9, registered to
# http://hanzo-git.hanzo.svc), every one mapping to
# docker://catthehacker/ubuntu:act-24.04. github.com has never had a runner
# carrying it — the org has zero runners at any level, because the ARC scale set
# that once served this label is gone (no `autoscalingrunnersets` CRD remains in
# the cluster).
#
# While this file sat in `.github/workflows/`, BOTH forges ran it: the forge
# reads `.hanzo/workflows` AND `.github/workflows`, so every tag started two
# builds — one that worked and one that could never start. The GitHub copy
# queued against a label nothing offered and sat there (v1.9.13 waited 17 hours
# before it was cancelled). Moving the file leaves exactly one execution, on the
# forge, because GitHub reads only `.github/workflows`. One tag, one build.
#
# NEVER set `runs-on: ubuntu-latest` here. That label was removed from the pool
# deliberately after an upstream workflow wedged all 10 runners; a job asking for
# it queues for its full 24h timeout and reports NO error. Use the fleet labels.
#
# Every `hanzo` platform binary is cross-built from that ONE Linux pool — no
# macOS or Windows runner exists, or is needed. `cargo-zigbuild` uses Zig as the
# cross-linker, so a single amd64 host emits linux-{amd64,arm64}, windows-amd64
# and macos-{amd64,arm64}. That is what makes a docker-only, linux-only runner
# pool sufficient to ship all five targets.
#
# What made the matrix build "everywhere" finally possible:
#   - reqwest on rustls (OpenSSL gone) — no per-arch OpenSSL headers.
#   - the credential store dropped Linux secret-service (no `libdbus` C dep) and
#     falls back to an owner-only file; the native keychain is compiled only for
#     macOS/Windows, whose system frameworks Zig links against directly.
#   - macOS needs a vendored SDK (Zig ships no macOS frameworks): the apple jobs
#     fetch a pinned, checksum-verified MacOSX SDK and point `SDKROOT` at it.
#
# A target that cannot build must be ABSENT, not listed-and-failing: a red release
# is indistinguishable from a broken one, and a pending target wedged the matrix
# before v1.2.0. Every target here produces a real binary from this pool.

on:
  workflow_dispatch:
  push:
    tags: ['v[0-9]+.[0-9]+.[0-9]+']

permissions:
  contents: write

env:
  ZIG_VERSION: '0.13.0'
  ZIG_SHA256: 'd45312e61ebcc48032b77bc4cf7fd6915c11fa16e4aad116b66c9468211230ea'
  ZIGBUILD_VERSION: '0.23.0'
  MACOS_SDK_URL: 'https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.3.sdk.tar.xz'
  MACOS_SDK_SHA256: 'cd4f08a75577145b8f05245a2975f7c81401d75e9535dcffbb879ee1deefcbf4'

jobs:
  build:
    name: ${{ matrix.target.id }}
    runs-on: hanzo-build-linux-amd64
    strategy:
      fail-fast: false
      matrix:
        target:
          - { id: linux-amd64,   rust: x86_64-unknown-linux-gnu,  bin: hanzo,     host: true }
          - { id: linux-arm64,   rust: aarch64-unknown-linux-gnu, bin: hanzo }
          - { id: windows-amd64, rust: x86_64-pc-windows-gnu,      bin: hanzo.exe }
          - { id: darwin-amd64,  rust: x86_64-apple-darwin,        bin: hanzo,     apple: true }
          - { id: darwin-arm64,  rust: aarch64-apple-darwin,       bin: hanzo,     apple: true }
    steps:
      - uses: actions/checkout@v4

      # A tag is a claim about what the binary IS. v1.3.0 once shipped a 1.2.0
      # binary because nothing checked the tag against the crate. Refuse to
      # publish a mislabeled release.
      - name: Tag matches the crate
        run: |
          set -euo pipefail
          v="$(grep -m1 '^version' Cargo.toml | sed -E 's/.*"(.*)".*/\1/')"
          [ "v$v" = "${{ github.ref_name }}" ] || {
            echo "tag ${{ github.ref_name }} but Cargo.toml is $v — the release would be mislabeled"
            exit 1
          }
          echo "tag matches crate: v$v"

      # Zig IS the C toolchain now. `build-essential` is only for the host link
      # (`cargo install`, the host test); OpenSSL and dbus dev packages are GONE
      # because nothing links them anymore. curl+jq publish; xz extracts Zig+SDK.
      - name: Toolchain deps
        run: |
          set -euo pipefail
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends build-essential curl jq tar xz-utils

      # `toolchain: stable` is STATED, not inferred from the `@stable` ref.
      # dtolnay/rust-toolchain reads the branch name out of `github.action_ref`
      # to decide the toolchain — that is GitHub-runner behaviour, and act_runner
      # on our own forge does not reproduce it, so the action received an empty
      # input and failed with "'toolchain' is a required input".
      #
      # It only ever broke ONE target, which is why it looked like a linux-amd64
      # problem: linux-amd64 is the sole `host: true` entry, so it alone runs
      # `make test` -> the drift gate, and the gate's own workflow carried the
      # same step. Four targets shipped, the fifth did not, and the release
      # looked healthy while missing the only binary our own amd64 sandbox can
      # run.
      - uses: dtolnay/rust-toolchain@stable
        with:
          toolchain: stable
          targets: ${{ matrix.target.rust }}

      - name: Zig (the cross-linker)
        run: |
          set -euo pipefail
          curl -fsSL "https://ziglang.org/download/${ZIG_VERSION}/zig-linux-x86_64-${ZIG_VERSION}.tar.xz" -o zig.tar.xz
          echo "${ZIG_SHA256}  zig.tar.xz" | sha256sum -c -
          mkdir -p "$HOME/zig"
          tar -xf zig.tar.xz -C "$HOME/zig" --strip-components=1
          echo "$HOME/zig" >> "$GITHUB_PATH"

      - name: cargo-zigbuild
        run: cargo install cargo-zigbuild --version "${ZIGBUILD_VERSION}" --locked

      # Zig ships no macOS frameworks (Security/CoreServices/CoreFoundation, which
      # the native keychain links). Vendor a pinned SDK and point SDKROOT at it.
      - name: Vendor macOS SDK
        if: matrix.target.apple
        run: |
          set -euo pipefail
          curl -fsSL "${MACOS_SDK_URL}" -o macsdk.tar.xz
          echo "${MACOS_SDK_SHA256}  macsdk.tar.xz" | sha256sum -c -
          tar -xf macsdk.tar.xz -C "$HOME"
          echo "SDKROOT=$HOME/MacOSX11.3.sdk" >> "$GITHUB_ENV"

      # Tests GATE the artifact, and THE DRIFT GATE IS ONE OF THEM. The command
      # tree is a CLAIM about what the server serves, and a release is the moment
      # that claim reaches users — so `tests/spec_drift.rs` puts it to the evidence
      # in BOTH directions before one artifact is minted: no command addressing a
      # route the server denies, no served product no command reaches, and no
      # hand-edited artifact anywhere in the derivation chain. v1.7.2 shipped 149
      # `/v1/cloud/<verb-noun>` commands the server has never served, a later spec
      # carried 129 more, and 46 served products shipped with no command at all.
      #
      # There was a separate `make verify` step here, ahead of this one, from when
      # that gate needed the network and could not live in a test. It is hermetic
      # now — it rules on the checked-in `spec/live.json` — so it runs HERE, once.
      # Two steps running one predicate is two places to keep it true.
      #
      # HOST arch only: a cross-built binary cannot execute here, and the cross
      # targets are the same source through the same compiler.
      #
      # ALL targets, not `--bin hanzo`: that selector runs only the binary's unit
      # tests, so `tests/spec_drift.rs` compiled here and never once executed.
      #
      # `make test`, not `cargo test` — and that is not a style choice. The gates
      # in `tests/` EXEC `driftgate` and `genproduct`, which sit behind
      # `required-features = ["maintainer"]`, so a bare `cargo test` never builds
      # them and both gates die on `NotFound` instead of ruling. Measured: under a
      # hand-edit to `generated.rs`, `cargo test --locked` printed the identical
      # two NotFound panics it prints on a CLEAN tree — its verdict did not depend
      # on its subject at all. The Makefile owns the one invocation that builds the
      # tools first and runs the whole workspace; a second spelling here is a
      # second thing to keep true, and this is the copy that guards the artifact.
      - name: Test — including the drift gate, in both directions
        if: matrix.target.host
        run: make test

      - name: Build
        run: cargo zigbuild --release --bin hanzo --locked --target ${{ matrix.target.rust }}

      - name: Package
        run: |
          set -euo pipefail
          out="target/${{ matrix.target.rust }}/release/${{ matrix.target.bin }}"
          test -f "$out"
          asset="hanzo-${{ matrix.target.id }}.tar.gz"
          tar -czf "$asset" -C "$(dirname "$out")" "${{ matrix.target.bin }}"
          sha256sum "$asset" > "$asset.sha256"
          echo "packaged $asset"

      # Publish STRAIGHT from the build job, over the REST API.
      #
      # No artifact round-trip: the org's artifact storage quota is exhausted and
      # it failed builds that had already succeeded. The artifact was only ever a
      # courier; the release is the destination.
      #
      # curl, not `gh`: the runner image ships no gh CLI. curl+jq is what the
      # runner actually has, so publish depends on nothing beyond what the build
      # needs.
      #
      # GH_PAT, not `github.token`. This job runs on git.hanzo.ai, so
      # `github.token` is a FORGE token — and these two URLs are github.com.
      # api.github.com rejected it, both the GET and the POST returned no `.id`,
      # and the step died on its own diagnostic: `could not create or find
      # release v1.9.16`. Every step above it had already passed; the release
      # was built and then dropped on the floor at the last line. v1.9.13,
      # v1.9.15 and v1.9.16 all failed exactly here, which is why users are
      # still on v1.9.12.
      #
      # GH_PAT is an org secret at git.hanzo.ai/hanzoai (repo-level secrets are
      # empty; org secrets inherit) — the same one cloud publishes with. BUILD
      # happens on the forge; DISTRIBUTION stays on GitHub Releases because
      # hanzo.sh resolves api.github.com/repos/hanzoai/cli/releases/latest.
      # Crossing that boundary is the one thing this step needs a credential for.
      - name: Publish
        env:
          TOKEN: ${{ secrets.GH_PAT }}
          TAG: ${{ github.ref_name }}
          REPO: ${{ github.repository }}
        run: |
          set -euo pipefail
          api="https://api.github.com/repos/$REPO"
          uploads="https://uploads.github.com/repos/$REPO"
          auth=(-H "Authorization: Bearer $TOKEN" -H "Accept: application/vnd.github+json")

          # Find-or-create: parallel matrix jobs share ONE release, and a re-run
          # must upload, not fail on an existing release.
          id="$(curl -s "${auth[@]}" "$api/releases/tags/$TAG" | jq -r '.id // empty')"
          if [ -z "$id" ]; then
            id="$(curl -s -X POST "${auth[@]}" \
              -d "$(jq -nc --arg t "$TAG" '{tag_name:$t,name:$t,generate_release_notes:true}')" \
              "$api/releases" | jq -r '.id // empty')"
          fi
          # A concurrent job may have created it between our GET and POST; re-read.
          if [ -z "$id" ]; then
            id="$(curl -s "${auth[@]}" "$api/releases/tags/$TAG" | jq -r '.id // empty')"
          fi
          [ -n "$id" ] || { echo "could not create or find release $TAG"; exit 1; }

          for f in "hanzo-${{ matrix.target.id }}.tar.gz" "hanzo-${{ matrix.target.id }}.tar.gz.sha256"; do
            test -f "$f"
            # Same name twice would 422; drop the old asset first so re-runs work.
            old="$(curl -s "${auth[@]}" "$api/releases/$id/assets" | jq -r --arg n "$f" '.[] | select(.name==$n) | .id')"
            [ -n "$old" ] && curl -s -X DELETE "${auth[@]}" "$api/releases/assets/$old" >/dev/null
            code="$(curl -s -o /tmp/up.json -w '%{http_code}' -X POST "${auth[@]}" \
              -H "Content-Type: application/octet-stream" \
              --data-binary @"$f" "$uploads/releases/$id/assets?name=$f")"
            [ "$code" = "201" ] || { echo "upload $f failed ($code)"; cat /tmp/up.json; exit 1; }
            echo "uploaded $f"
          done

  # PUBLISH TO NPM, and only after every target uploaded.
  #
  # `needs: build` is the whole safety property: the npm package's postinstall
  # downloads `hanzo-<platform>.tar.gz` from the release for its own version, so
  # publishing before the assets exist ships a package whose install 404s. v1.9.41
  # is the cautionary case — four targets uploaded, linux-amd64 did not, and the
  # release read as green while the only binary our own amd64 cluster runs was
  # absent. A matrix job that fails now blocks this one.
  #
  # One artifact, not six: the binary is the release asset, and npm carries a
  # downloader rather than five per-platform packages. Five packages would be a
  # second copy of the same bytes under a second set of version numbers, and the
  # copies drift.
  npm:
    name: npm
    needs: build
    if: startsWith(github.ref, 'refs/tags/v')
    runs-on: hanzo-build-linux-amd64
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: '22'
          registry-url: 'https://registry.npmjs.org'
      - name: Version from the tag, never from the file
        run: |
          set -euo pipefail
          v="${GITHUB_REF_NAME#v}"
          # The tag is the authority. package.json carrying a different number is
          # how a release publishes bytes under a version nobody built.
          node -e '
            const fs=require("fs"),p="npm/package.json";
            const j=JSON.parse(fs.readFileSync(p));
            j.version=process.argv[1];
            fs.writeFileSync(p,JSON.stringify(j,null,2)+"\n");
          ' "$v"
          echo "publishing @hanzo/cli@$v"
      # THE CREDENTIAL IS READ FROM KMS, WHICH IS WHERE A SECRET LIVES. This job
      # took `secrets.NPM_TOKEN`, and that secret exists on neither the hanzoai
      # org nor this repo on git.hanzo.ai. An absent secret interpolates to the
      # empty string rather than failing, so npm was handed a blank credential,
      # answered `ENEEDAUTH`, and every release since the job was written failed
      # here after building all five binaries — which is why @hanzo/cli is still
      # a 404 on the registry. Same identity and same path hanzoai/ui-design
      # reads, so there is one npm credential in the fleet and nothing to rotate.
      #
      # python3, not jq: jq is installed by the build job above, and this job
      # does not run that step.
      - name: Publish
        working-directory: npm
        env:
          KMS_CLIENT_ID: ${{ secrets.KMS_CLIENT_ID }}
          KMS_CLIENT_SECRET: ${{ secrets.KMS_CLIENT_SECRET }}
          KMS: ${{ vars.KMS_ENDPOINT || 'https://kms.hanzo.ai' }}
        run: |
          set -euo pipefail
          val() { python3 -c 'import json,sys;print(json.load(sys.stdin).get(sys.argv[1],""))' "$1"; }
          kt=$(curl -fsS "$KMS/v1/kms/auth/login" -H 'Content-Type: application/json' \
                 -d "{\"clientId\":\"$KMS_CLIENT_ID\",\"clientSecret\":\"$KMS_CLIENT_SECRET\"}" | val accessToken)
          [ -n "$kt" ] || { echo "::error::KMS login failed at $KMS/v1/kms/auth/login (check KMS_CLIENT_ID/KMS_CLIENT_SECRET)"; exit 1; }
          # No -f on the read: a miss must name the path that was empty rather
          # than exiting 22 with nothing to act on.
          NODE_AUTH_TOKEN=$(curl -sS "$KMS/v1/kms/secrets/NPM_TOKEN?env=prod" -H "Authorization: Bearer $kt" | val value)
          [ -n "$NODE_AUTH_TOKEN" ] || { echo "::error::no NPM_TOKEN at $KMS/v1/kms/secrets/NPM_TOKEN?env=prod"; exit 1; }
          echo "::add-mask::$NODE_AUTH_TOKEN"
          export NODE_AUTH_TOKEN
          # --ignore-scripts: postinstall fetches a binary, and it must run on the
          # INSTALLING machine, not on this runner where it would bake one
          # platform's binary into the tarball.
          npm publish --access public --ignore-scripts

  # BUMP THE HOMEBREW TAP, because nothing ever did.
  #
  # `brew install hanzoai/tap/hanzo` served v1.9.18 while every other install path
  # served the current build: the formula was hand-written in three commits on one
  # day and then sat through 35 releases. The tap's own gate cannot catch that —
  # it downloads each url and checks it against the sha256 beside it, which stays
  # green forever on a stale but self-consistent pair. Freshness is not a fact
  # about that repo, it is a fact about THIS release, so the bump belongs here.
  #
  # `needs: build` for the same reason npm has it: the formula names assets, and a
  # formula pointing at a release whose upload failed is a broken install.
  #
  # FOUR targets, not five — Homebrew has no windows.
  #
  # The checksums are the `.sha256` sidecars the build job already uploaded, never
  # recomputed here. Recomputing means downloading the asset and trusting a second
  # reading of it; reading the sidecar means the tap and the release cannot
  # disagree, and the formula's own `v#{version}` urls make a release ONE version
  # line and four checksums.
  #
  # It writes the FORGE, and the forge's push mirror carries it to github.com,
  # where `brew tap` clones from. Writing github directly is what the first run
  # did, and it left git.hanzo.ai — the canonical copy — a commit behind, with a
  # mirror push aimed at a github main it no longer contained. So the credential
  # is FORGE_TOKEN, the same one the spec re-pin pushes this repo's own protected
  # main with; `secrets.GH_PAT` still reaches github, but only the release assets
  # need it. The sidecars below are read over https from github Releases, which is
  # where DISTRIBUTION lives either way.
  tap:
    name: homebrew tap
    needs: build
    if: startsWith(github.ref, 'refs/tags/v')
    runs-on: hanzo-build-linux-amd64
    steps:
      - name: Point the formula at this release
        env:
          FORGE_TOKEN: ${{ secrets.FORGE_TOKEN }}
          TAG: ${{ github.ref_name }}
          REPO: ${{ github.repository }}
        run: |
          set -euo pipefail
          [ -n "${FORGE_TOKEN:-}" ] || { echo "::error::the tap bump pushes hanzoai/homebrew-tap and needs FORGE_TOKEN"; exit 1; }
          v="${TAG#v}"
          tap_url="${GITHUB_SERVER_URL:-https://git.hanzo.ai}/hanzoai/homebrew-tap"
          git clone --depth 1 "$tap_url" tap
          git -C tap config --local http.extraheader "Authorization: token $FORGE_TOKEN"
          f=tap/Formula/hanzo.rb
          test -f "$f"

          sed -i "s/^  version \".*\"$/  version \"$v\"/" "$f"
          for t in darwin-arm64 darwin-amd64 linux-arm64 linux-amd64; do
            sha="$(curl -fsSL "https://github.com/$REPO/releases/download/$TAG/hanzo-$t.tar.gz.sha256" | cut -d' ' -f1)"
            [ ${#sha} -eq 64 ] || { echo "::error::no sha256 sidecar for hanzo-$t at $TAG"; exit 1; }
            # Anchored on the asset name, so each checksum lands under its own url.
            perl -0pi -e "s{(hanzo-$t\.tar\.gz\"\s+sha256 \")[0-9a-f]{64}}{\${1}$sha}" "$f"
            # An anchor that stops matching — a renamed asset, a reflowed formula —
            # leaves the PREVIOUS release's checksum under the new url and reports
            # nothing. That ships a formula whose every install fails on a checksum
            # mismatch, so the rewrite is read back rather than assumed.
            grep -q "\"$sha\"" "$f" || { echo "::error::hanzo-$t checksum never landed in the formula"; exit 1; }
          done

          grep -q "^  version \"$v\"\$" "$f" || { echo "::error::formula kept its old version"; exit 1; }

          cd tap
          if git diff --quiet; then echo "formula is already at $v"; exit 0; fi
          git -c user.name=hanzo-dev -c [email protected] commit -qam "hanzo $v"
          git push -q "$tap_url" HEAD:main
          echo "tap now at $v"