chat/.hanzo/workflows/deploy.yml

# Build the image and push it. Nothing else.
#
# The previous version called buildctl-daemonless.sh, which does not exist in
# the job image (catthehacker/ubuntu:act-24.04, per GIT_RUNNER_LABELS) — exit
# 127 — and read secrets.GIT_CLONE_TOKEN, which exists on neither the repo nor
# the hanzoai org. It had never produced an image.
#
# No deploy step on purpose: kubectl is absent from the job image too, and a
# CI-side patch is reverted within minutes by cd.hanzo.ai's selfHeal, which
# restores the App CR from the reviewed pin in hanzoai/universe. CI's job ends
# at "the tag exists and resolves"; choosing which tag runs is a reviewed change.
#
# This matters more than it used to: actions-runner-controller was decommissioned
# 2026-07-27, so this pipeline is now the ONLY CI this repo has.
name: deploy
on:
  push:
    branches: [main]
    # Prose does not get a production outage. This lane builds an image, bumps
    # the version, pins hanzoai/universe and cd rolls it — and the deploy is
    # `replicas: 1, strategy: Recreate`, so every roll costs ~3 minutes of
    # hanzo.chat answering 503. Paying that to change a sentence in LLM.md is
    # not a tradeoff anyone chose; it was just never filtered.
    #
    # `**.md` is safe here, checked rather than assumed: nothing imports a
    # markdown file into the client or the server bundle, and all 31 tracked
    # `.md` are docs, issue templates or migration READMEs. `.hanzo/workflows/**`
    # is deliberately NOT ignored — a change to this lane must prove it still
    # builds.
    #
    # An EMPTY commit does not retrigger this lane. The filter matches on the
    # paths a push touches, and a commit that touches none has nothing to match,
    # so `--allow-empty` reads like a retry and fires no run at all. Rebuild a
    # commit by changing a real file, or dispatch the workflow.
    paths-ignore:
      - '**.md'
  workflow_dispatch:

concurrency:
  group: deploy-chat-${{ github.ref }}
  cancel-in-progress: true

# `contents: write` is for the release TAG. It is not enough for a commit to
# main and never was — Hanzo Git refuses an Actions-token push to a protected
# branch whatever this says, which is why the write-back is gone.
permissions:
  contents: write
  packages: write

jobs:
  # Measures whether the multi-stage build still works, and what it would save.
  # Dispatch only: it never runs on a normal deploy, so it costs nothing and
  # cannot affect what ships.
  #
  # Why it exists: chat mounts an RWO volume holding a SQLite database with an
  # in-process writer, so `strategy: Recreate` is correct and every roll is a
  # real outage. Most of that window is the pull — 658 MB took 89 seconds on the
  # last roll. The single-stage Dockerfile ships pnpm, python3, pip, uv and every
  # devDependency into the runtime image, and Dockerfile.multi was written to fix
  # exactly that but has drifted eight commits behind and is wired to nothing.
  #
  # This answers the two questions that decide whether closing that gap is worth
  # anyone's afternoon: does it still build, and how much smaller is it. It does
  # NOT push, and nothing deploys from it — a red run here means the multi-stage
  # file needs work, not that chat is broken.
  multi-stage-size:
    if: github.event_name == 'workflow_dispatch'
    runs-on: [hanzo-build-linux-amd64]
    steps:
      - uses: actions/checkout@v4

      # No PUBLISHABLE_KEY: this image is measured and discarded, never served.
      # Dockerfile.multi predates the key gate, so it will build without one —
      # that gap is part of what the port has to close.
      #
      # The same two commands the build job runs, for the same reason: a wrapper
      # fetched from github.com is a network dependency this runner does not
      # need to take.
      - name: Build Dockerfile.multi
        continue-on-error: true
        run: |
          set -uo pipefail
          docker buildx create --name hanzo-measure --driver docker-container --use 2>/dev/null \
            || docker buildx use hanzo-measure
          docker buildx build . \
            --file ./Dockerfile.multi \
            --platform linux/amd64 \
            --tag chat-multi:measure \
            --load

      - name: Report the size, against what runs today
        run: |
          set -uo pipefail
          if ! docker image inspect chat-multi:measure >/dev/null 2>&1; then
            echo "::warning::Dockerfile.multi did not build — it is eight commits behind Dockerfile and needs the port before it can replace it"
            exit 0
          fi
          multi=$(docker image inspect chat-multi:measure --format '{{.Size}}')
          echo "Dockerfile.multi: $((multi / 1000000)) MB"
          echo "shipping today:   658 MB (measured from the 1.0.220 pull)"
          echo "pull time is the outage: every roll tears the only pod down before the new one starts."

  build:
    # Must name a label a runner ACTUALLY advertises. The git-runners offer
    # hanzo-build-linux-amd64; a label nothing offers does not fail, it queues
    # until the 24h timeout — silence, not an error.
    runs-on: [hanzo-build-linux-amd64]
    steps:
      # Full history + tags: this job TAGS the release, so it needs the tag
      # namespace to push into and a non-shallow clone to describe from.
      #
      # THE LAST github.com FETCH IN THIS JOB, and the only one left because it
      # is the only one that is not a wrapper: it clones with a credential the
      # runner supplies, and getting that wrong fails every build rather than
      # one. The three docker/* steps it used to sit beside are gone — each was
      # one command (`docker login`, `buildx create`, `buildx build`) that the
      # runner already has.
      #
      # The hop matters because it is where the lane dies when egress does:
      # `dial tcp …:443: i/o timeout` here, before install, before build, with
      # nothing in the tree read. Four attempts on one commit failed that way
      # while cicd — which imports a forge-hosted reusable,
      # hanzoai/ci/.hanzo/workflows/build.yml@v1 — went green beside them.
      # Mirroring actions/checkout onto git.hanzo.ai closes the rest.
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true

      # The version universe PINS. We don't ship shas: a sha names a commit, and
      # a CR that names a commit cannot be read as "which release is live". The
      # number is derived from package.json against the registry floor, never
      # typed — one implementation (hanzoai/ci bin/imgver), every builder.
      - id: ver
        uses: hanzoai/ci/.github/actions/imgver@v1
        with: { repo: ghcr.io/hanzoai/chat }
        env: { GH_PAT: '${{ secrets.GH_PAT }}' }

      # imgver reads package.json as a FLOOR — max(declared, published) + 1 — so
      # a declared number nothing writes back sinks one patch per release. It
      # reached 1.0.77 against a published 1.0.113 that way: 36 patches behind,
      # and silent, because a stale floor breaks nothing. It just loses.
      #
      # Stamping before the build is what makes the TAG below honest: the tree
      # that gets tagged is byte-identical to the tree that was built.
      - name: Stamp package.json with the release version
        run: |
          set -euo pipefail
          v="${{ steps.ver.outputs.version }}"
          sed -i.bak -E "s/(\"version\"[[:space:]]*:[[:space:]]*\")[^\"]+(\")/\1${v}\2/" package.json && rm -f package.json.bak
          echo "package.json -> $(grep -m1 '\"version\"' package.json)"

      # Kept alongside the semver, for forensics only. Nothing pins it.
      - id: meta
        run: echo "tag=sha-$(echo "$GITHUB_SHA" | cut -c1-7)" >> "$GITHUB_OUTPUT"

      # Same builder the action creates, by the command it runs. `network=host`
      # so the build can reach the registry cache and the runner's own mirrors.
      - name: Prepare the builder
        run: |
          set -euo pipefail
          docker buildx create --name hanzo --driver docker-container \
            --driver-opt network=host --use 2>/dev/null || docker buildx use hanzo
          docker buildx inspect --bootstrap >/dev/null

      # Without this the push is unauthenticated and 403s at the registry.
      #
      # `docker login` directly, not docker/login-action: every action in this
      # lane is fetched from github.com at run time, and that hop is the one
      # thing here that leaves the estate. It is also where the lane dies when
      # egress does — four builds on one commit failed cloning actions/checkout,
      # before install, with nothing in the tree read. A wrapper around one
      # command is not worth a network dependency.
      # Retried, because the action this replaces retries and the runner's
      # egress is not reliable: a single attempt met
      # `Get "https://ghcr.io/v2/": context deadline exceeded` and failed the
      # build on a network blip. Backs off, and says which attempt it is so a
      # log shows a flaky link rather than a wrong credential.
      - name: Sign in to the registry
        run: |
          set -uo pipefail
          for i in 1 2 3 4 5; do
            if printf '%s' "${{ secrets.GHCR_TOKEN }}" \
                 | docker login ghcr.io -u "${{ secrets.GHCR_USER }}" --password-stdin; then
              exit 0
            fi
            echo "registry sign-in attempt $i failed; retrying"
            sleep $((i * 5))
          done
          echo "::error::could not sign in to ghcr.io after 5 attempts"
          exit 1

      # The publishable ingest key, from KMS — the only store secrets come from.
      # Vite inlines import.meta.env at BUILD, so this cannot be delivered at
      # runtime: an unkeyed bundle sends every beacon down cloud's anonymous lane,
      # where track/identify/group are dropped and still answered 200.
      #
      # Same KMS surface and same key name the shared hanzoai/ci reusable uses for
      # `build_secrets`, so the key has ONE name and ONE home no matter which
      # pipeline builds. This repo builds bespoke (it is not on the reusable), so
      # the fetch is spelled out here.
      #
      # Absence is ANNOUNCED, not swallowed: it is the exact silent degradation
      # this key exists to fix. Non-fatal on purpose — a KMS hiccup must not block
      # shipping the product — but it lands as a red annotation on the run.
      - name: Fetch the ingest key from KMS
        id: ingest
        env:
          KMS_CLIENT_ID: ${{ secrets.KMS_CLIENT_ID }}
          KMS_CLIENT_SECRET: ${{ secrets.KMS_CLIENT_SECRET }}
          KMS_ENDPOINT: ${{ vars.KMS_ENDPOINT || 'https://api.hanzo.ai' }}
        # NOT the default shell. The runner invokes `bash --noprofile --norc -e -o
        # pipefail`, and `set` only ADDS options — a script cannot take `-e` back
        # off. So every careful `exit 0` below was unreachable: `curl -sf` answers
        # non-zero on any non-2xx, `-e` killed the step at the first one, and the
        # run showed a bare red X 0.19s in with not one word of why. Every
        # diagnostic this step exists to print was dead code.
        shell: bash --noprofile --norc {0}
        run: |
          set -uo pipefail
          if [ -z "${KMS_CLIENT_ID:-}" ] || [ -z "${KMS_CLIENT_SECRET:-}" ]; then
            echo "::error::KMS credentials absent — shipping a bundle with no ingest key; every track/identify/group it emits will be dropped"
            exit 0
          fi
          # Status and body separately, so a failure can SAY which one it was.
          # `curl -sf` collapses 401, 404 and a network drop into one empty string,
          # which is how "login failed" and "secret absent" became the same
          # non-event.
          #
          # -m 30 because a hung connection has no status to report: without it a
          # KMS that accepts the socket and never answers holds the runner until
          # the job timeout, and the release just never happens. A timeout lands
          # here as code 000 and takes the same diagnosed path as any other
          # failure. The "Ship it" step below already reads KMS this way.
          login=$(curl -s -m 30 -w '\n%{http_code}' "$KMS_ENDPOINT/v1/kms/auth/login" \
            -H 'Content-Type: application/json' \
            -d "{\"clientId\":\"$KMS_CLIENT_ID\",\"clientSecret\":\"$KMS_CLIENT_SECRET\"}")
          code=$(printf '%s' "$login" | tail -n1)
          TOKEN=$(printf '%s' "$login" | sed '$d' | jq -r '.accessToken // empty')
          if [ -z "$TOKEN" ]; then
            echo "::error::KMS login answered $code — the CI credentials no longer authenticate. Bundle ships with no ingest key."
            exit 0
          fi
          # KMS is embedded in hanzoai/cloud now (HIP-0106): the org comes from the
          # TOKEN, not the url, and the body is flat {"name","env","value"}. The old
          # /v1/kms/orgs/<org>/... + .secret.value pair 404s and selects nothing.
          # The key lives at hanzo / env=prod / path=deploy / PUBLISHABLE_KEY, and
          # that address is load-bearing: a key stored anywhere else is never read.
          get=$(curl -s -m 30 -w '\n%{http_code}' "$KMS_ENDPOINT/v1/kms/secrets/deploy/PUBLISHABLE_KEY?env=prod" \
            -H "Authorization: Bearer $TOKEN")
          code=$(printf '%s' "$get" | tail -n1)
          V=$(printf '%s' "$get" | sed '$d' | jq -r '.value // empty')
          if [ -z "$V" ]; then
            echo "::error::PUBLISHABLE_KEY read answered $code (hanzo/deploy env=prod) — shipping a bundle with no ingest key"
            exit 0
          fi
          echo "::add-mask::$V"
          echo "key=$V" >> "$GITHUB_OUTPUT"
          echo "ingest key resolved ($code, ${#V} chars)"

      # `docker buildx build`, the command the action wraps. GIT_AUTH_TOKEN is
      # not carried over: the Dockerfile mounts no secret, so it was passed and
      # discarded. PUBLISHABLE_KEY stays a build-arg because Vite inlines it.
      # Same reason as the sign-in above: the push crosses the same link, and
      # the action this replaces retries where a bare command gives up.
      - name: Build and push
        run: |
          set -uo pipefail
          build() { docker buildx build . \
            --file ./Dockerfile \
            --platform linux/amd64 \
            --provenance=false \
            --build-arg PUBLISHABLE_KEY="${{ steps.ingest.outputs.key }}" \
            --build-arg BUILD_SHA="${{ github.sha }}" \
            --tag ghcr.io/hanzoai/chat:${{ steps.ver.outputs.version }} \
            --tag ghcr.io/hanzoai/chat:${{ steps.meta.outputs.tag }} \
            --push; }
          for i in 1 2 3; do
            build && exit 0
            echo "build/push attempt $i failed; retrying"
            sleep $((i * 10))
          done
          echo "::error::build and push failed after 3 attempts"
          exit 1

      # build-push-action can exit 0 before the manifest resolves, so a green run
      # could still mean a future ImagePullBackOff. Prove it pulls.
      - name: Verify the pushed image resolves
        run: |
          set -euo pipefail
          img="ghcr.io/hanzoai/chat:${{ steps.ver.outputs.version }}"
          for i in 1 2 3 4 5 6; do
            docker buildx imagetools inspect "$img" >/dev/null 2>&1 && { echo "resolved $img"; exit 0; }
            sleep 5
          done
          echo "::error::pushed image never resolved: $img"; exit 1

      # THE STEP THAT SHIPS. Everything above builds and proves an image; none of
      # it deploys one. cd.hanzo.ai reconciles from universe's values files and
      # never reads the registry, so a chat image that is built, pushed and
      # resolvable is still unreachable until this pin moves. That is why the
      # fleet sat on 1.0.115 while newer images existed: the lane ended at "push"
      # and waited for a human who had no reason to know they were the mechanism.
      #
      # It runs BEFORE the tag below on purpose. Tagging is bookkeeping and is
      # deliberately last so it cannot un-ship a good build; this one IS the
      # ship, and a release that tags a version it never deployed is the same lie
      # in the other direction.
      #
      # pin.sh resolves tag AND digest from ONE registry lookup and writes them
      # together. Two lookups at two moments drift, and drift here does not fail
      # — it succeeds at doing nothing, because a reference carrying both
      # resolves by DIGEST: the pod reports the new version and serves the old
      # bytes.
      - name: Ship it
        env:
          KMS_CLIENT_ID: ${{ secrets.KMS_CLIENT_ID }}
          KMS_CLIENT_SECRET: ${{ secrets.KMS_CLIENT_SECRET }}
          KMS_ENDPOINT: ${{ vars.KMS_ENDPOINT || 'https://api.hanzo.ai' }}
          KMS_SECRET_ENV: ${{ vars.KMS_SECRET_ENV || 'prod' }}
          # pin.sh probes the registry before it moves anything; these let it read
          # a private manifest instead of falling back to anonymous.
          GHCR_USER: ${{ secrets.GHCR_USER }}
          GHCR_TOKEN: ${{ secrets.GHCR_TOKEN }}
        run: |
          set -euo pipefail
          VERSION="${{ steps.ver.outputs.version }}"

          # Secrets come from KMS, never from a file or a repo variable.
          KMS_TOKEN=$(curl -fsS "${KMS_ENDPOINT}/v1/kms/auth/login" \
            -H 'Content-Type: application/json' \
            -d "{\"clientId\":\"${KMS_CLIENT_ID}\",\"clientSecret\":\"${KMS_CLIENT_SECRET}\"}" \
            | jq -r '.accessToken // empty')
          [ -n "$KMS_TOKEN" ] || { echo "::error::KMS login failed at ${KMS_ENDPOINT} — image ${VERSION} is pushed but NOT deployed"; exit 1; }

          # The org is carried by the credential, not by the path. An /orgs/<org>/
          # spelling 404s, and curl -f turns that into a bare exit 22 that reads
          # like a missing secret rather than a wrong route.
          PIN_TOKEN=$(curl -fsS \
            "${KMS_ENDPOINT}/v1/kms/secrets/deploy/UNIVERSE_PIN_TOKEN?env=${KMS_SECRET_ENV}" \
            -H "Authorization: Bearer ${KMS_TOKEN}" | jq -r '.value // empty')
          [ -n "$PIN_TOKEN" ] || { echo "::error::UNIVERSE_PIN_TOKEN missing in KMS (path deploy, env ${KMS_SECRET_ENV}) — image ${VERSION} is pushed but NOT deployed"; exit 1; }
          echo "::add-mask::${PIN_TOKEN}"

          # cd reads THIS copy. universe also exists under hanzoai/ and is a
          # different, staler commit — pinning there deploys nothing, silently.
          git clone --quiet --depth 1 \
            "https://x-access-token:${PIN_TOKEN}@git.hanzo.ai/hanzo/universe" /tmp/universe

          # Another service can land its own pin between our clone and our push.
          # A shallow clone has no common history to rebase onto, so RE-APPLY
          # rather than merge: pinning is idempotent and derives everything from
          # the file it finds, so the retry re-checks the monotonic guard too.
          for attempt in 1 2 3 4 5; do
            /tmp/universe/charts/app/pin.sh chat "$VERSION"

            if git -C /tmp/universe push --quiet origin HEAD:main 2>/dev/null; then
              echo "pinned chat to ${VERSION} — cd.hanzo.ai rolls it out"; exit 0
            fi

            echo "universe moved under us (attempt ${attempt}/5) — re-applying on the new tip"
            git -C /tmp/universe fetch --quiet --depth 1 origin main
            git -C /tmp/universe reset --hard --quiet FETCH_HEAD
          done
          echo "::error::image ${VERSION} shipped but the pin never landed — chat is published and unreachable"; exit 1

      # ONLY after the image is proven pullable, so a git tag can never name a
      # version the registry lacks — which is what lets scripts/version.sh treat
      # git tags as a floor when it has no registry token.
      #
      # The tag also closes a gap of its own: this lane has never tagged, so the
      # tag stream stopped at v1.0.109 while images ran to 1.0.113. The fleet's
      # stated invariant is a git tag v<X.Y.Z> ⇔ an image :<X.Y.Z>; chat had the
      # images and not the tags.
      #
      # `[skip ci]` because this workflow triggers on every push to main. Both
      # remotes honour it — git.hanzo.ai via Gitea's SKIP_WORKFLOW_STRINGS
      # default, github.com natively — so the write-back cannot re-enter here.
      #
      # LAST: the image is pushed and resolvable by now, and universe pins the
      # image, not this tag. Bookkeeping must not be able to un-ship a good build.
      # hanzo.yml's `version-drift` gate is gone with the write-back — it existed
      # to prove that write-back landed, so keeping it would have meant a gate
      # that goes red forever about a copy with no authority.
      # TAG ONLY. The commit-to-main half of this step is gone, and the reason is
      # in the comment on `imgver` above: the version is `max(declared, published)
      # + 1`, so once an image is published the registry carries the floor and the
      # copy in package.json can only ever restate it. What the copy DOES buy — a
      # deliberate jump to, say, 1.1.0 — is a human editing that file, which max()
      # still honours. So the write-back kept a redundant copy in sync.
      #
      # It could not do even that: Hanzo Git refuses an Actions-token push to a
      # protected branch, so the five-attempt loop below retried a race it was
      # never losing, then failed the job. The image had already shipped, been
      # tagged and been pinned by then — a RED run that meant "released fine", the
      # same shape hanzo-apps/app resolved by deleting its write-back.
      #
      # The tag stays: `git push origin v${v}` succeeds (v1.0.150 is on Hanzo Git),
      # and the pre-build stamp stays too, so the tree that gets tagged is still
      # byte-identical to the tree that was built.
      - name: Tag the version this image ships
        run: |
          set -euo pipefail
          v="${{ steps.ver.outputs.version }}"
          git config user.name 'hanzo-ci'
          git config user.email '[email protected]'
          # Idempotent: this step gets RE-RUN, and `git tag` on an existing name
          # is fatal. A run that pushed its image and then died later leaves the
          # tag already there.
          if git rev-parse -q --verify "refs/tags/v${v}" >/dev/null; then
            echo "tag v${v} already exists"
          else
            git tag "v${v}"
            git push origin "v${v}"
          fi
          echo "v${v} tagged — package.json on main is not written back by design (see above)"