Pin the chart the publisher stands behind; drop the second deploy path
0.1.0 is the version the chart workflow's own comment names as never having matched its source, so every Application claimed a chart nobody could reproduce. 0.1.11 is what lux pins and what the registry actually holds. ci.yml applied manifests with kubectl on every push, beside an ApplicationSet that reconciles the same namespaces. Two delivery paths let the cluster disagree with git while both report success; CD is the one that stays. Claude-Session: https://claude.ai/code/session_01WAriF9qmzQJnBHfumZMxnd
This commit is contained in:
@@ -1,458 +0,0 @@
|
||||
# Universe CI -- Layer 2 Orchestrator
|
||||
#
|
||||
# Runs from .hanzo/workflows, which is the one directory this forge collects.
|
||||
# Gitea takes the FIRST of WORKFLOW_DIRS that exists in the pushed commit and
|
||||
# ignores the rest (modules/actions/workflows.go, listWorkflowsInDirs breaks on
|
||||
# the first hit), so this directory and .github/workflows were never going to
|
||||
# coexist: creating one switches the other off silently, with no run, no queued
|
||||
# job and no error. Everything this repo still wants moved here in one commit.
|
||||
#
|
||||
# Two modes:
|
||||
# 1. service-built (repository_dispatch): A single service was built + tested.
|
||||
# Update manifest, deploy that service, run smoke test.
|
||||
# 2. full-deploy (push/schedule/dispatch): Deploy all services from manifest,
|
||||
# run E2E.
|
||||
#
|
||||
# Layer 1 (build-service.yml): per-repo lint + unit test + build + push
|
||||
# Layer 2 (this): manifest tracking + deploy + integration E2E
|
||||
#
|
||||
# Zoo uses DOKS (do-sfo3-zoo-k8s) + GHCR (ghcr.io/zoolabs/).
|
||||
# Deploys via kubectl apply -k (kustomize) -- not Helm.
|
||||
|
||||
name: Universe CI
|
||||
|
||||
on:
|
||||
repository_dispatch:
|
||||
types: [service-built]
|
||||
|
||||
push:
|
||||
branches: [main, test, dev]
|
||||
paths:
|
||||
- 'manifests/**'
|
||||
- 'scripts/**'
|
||||
- '.hanzo/workflows/ci.yml'
|
||||
- 'e2e/**'
|
||||
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: "Deploy mode"
|
||||
type: choice
|
||||
options:
|
||||
- deploy-all
|
||||
- e2e-only
|
||||
default: deploy-all
|
||||
environment:
|
||||
description: "Target environment"
|
||||
type: choice
|
||||
options:
|
||||
- auto
|
||||
- main
|
||||
- test
|
||||
- dev
|
||||
default: auto
|
||||
|
||||
schedule:
|
||||
# Nightly full E2E at 03:00 UTC
|
||||
- cron: '0 3 * * *'
|
||||
|
||||
concurrency:
|
||||
group: universe-ci-${{ github.event.client_payload.branch || github.ref_name }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
CLUSTER: do-sfo3-zoo-k8s
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
id-token: write
|
||||
|
||||
jobs:
|
||||
# ============================================================
|
||||
# Resolve environment from trigger context
|
||||
# ============================================================
|
||||
resolve:
|
||||
name: Resolve Environment
|
||||
runs-on: [zoo-build-linux-amd64]
|
||||
outputs:
|
||||
environment: ${{ steps.env.outputs.environment }}
|
||||
namespace: ${{ steps.env.outputs.namespace }}
|
||||
mode: ${{ steps.env.outputs.mode }}
|
||||
service_key: ${{ steps.env.outputs.service_key }}
|
||||
service_sha: ${{ steps.env.outputs.service_sha }}
|
||||
service_image: ${{ steps.env.outputs.service_image }}
|
||||
service_app_name: ${{ steps.env.outputs.service_app_name }}
|
||||
steps:
|
||||
- name: Resolve
|
||||
id: env
|
||||
run: |
|
||||
EVENT="${GITHUB_EVENT_NAME}"
|
||||
|
||||
if [ "${EVENT}" = "repository_dispatch" ]; then
|
||||
BRANCH='${{ github.event.client_payload.branch }}'
|
||||
MODE="deploy-single"
|
||||
echo "service_key=${{ github.event.client_payload.service_key }}" >> "$GITHUB_OUTPUT"
|
||||
echo "service_sha=${{ github.event.client_payload.sha }}" >> "$GITHUB_OUTPUT"
|
||||
echo "service_image=${{ github.event.client_payload.image }}" >> "$GITHUB_OUTPUT"
|
||||
echo "service_app_name=${{ github.event.client_payload.app_name }}" >> "$GITHUB_OUTPUT"
|
||||
elif [ "${EVENT}" = "workflow_dispatch" ]; then
|
||||
BRANCH="${GITHUB_REF_NAME}"
|
||||
INPUT_ENV='${{ inputs.environment }}'
|
||||
if [ "${INPUT_ENV}" != "auto" ] && [ -n "${INPUT_ENV}" ]; then
|
||||
BRANCH="${INPUT_ENV}"
|
||||
fi
|
||||
MODE='${{ inputs.mode }}'
|
||||
if [ -z "${MODE}" ]; then MODE="deploy-all"; fi
|
||||
elif [ "${EVENT}" = "schedule" ]; then
|
||||
BRANCH="dev"
|
||||
MODE="e2e-only"
|
||||
else
|
||||
BRANCH="${GITHUB_REF_NAME}"
|
||||
MODE="deploy-all"
|
||||
fi
|
||||
|
||||
case "${BRANCH}" in
|
||||
main) NS="zoo-mainnet" ;;
|
||||
test) NS="zoo-testnet" ;;
|
||||
dev) NS="zoo-devnet" ;;
|
||||
*) NS="zoo-devnet" BRANCH="dev" ;;
|
||||
esac
|
||||
|
||||
echo "environment=${BRANCH}" >> "$GITHUB_OUTPUT"
|
||||
echo "namespace=${NS}" >> "$GITHUB_OUTPUT"
|
||||
echo "mode=${MODE}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
echo "Environment: ${BRANCH}"
|
||||
echo "Namespace: ${NS}"
|
||||
echo "Mode: ${MODE}"
|
||||
|
||||
# ============================================================
|
||||
# Mode: deploy-single -- update manifest + deploy one service
|
||||
# ============================================================
|
||||
update-manifest:
|
||||
name: Update Manifest
|
||||
runs-on: [zoo-build-linux-amd64]
|
||||
needs: resolve
|
||||
if: needs.resolve.outputs.mode == 'deploy-single'
|
||||
steps:
|
||||
- name: Fetch secrets from KMS
|
||||
id: kms
|
||||
env:
|
||||
KMS_CLIENT_ID: ${{ secrets.KMS_CLIENT_ID }}
|
||||
KMS_CLIENT_SECRET: ${{ secrets.KMS_CLIENT_SECRET }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
KMS_URL="${KMS_URL:-https://kms.hanzo.ai}"
|
||||
WORKSPACE="zoo-platform"
|
||||
|
||||
ACCESS_TOKEN="$(
|
||||
curl -fsS --max-time 10 -X POST "${KMS_URL}/api/v1/auth/universal-auth/login" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"clientId\":\"${KMS_CLIENT_ID}\",\"clientSecret\":\"${KMS_CLIENT_SECRET}\"}" \
|
||||
| jq -r '.accessToken'
|
||||
)"
|
||||
|
||||
[ -n "${ACCESS_TOKEN}" ] && [ "${ACCESS_TOKEN}" != "null" ] || { echo "::error::KMS auth failed"; exit 1; }
|
||||
|
||||
fetch_secret() {
|
||||
curl -fsS "${KMS_URL}/api/v3/secrets/raw/${1}?workspaceSlug=${WORKSPACE}&environment=prod&secretPath=/ci&viewSecretValue=true&include_imports=true" \
|
||||
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
|
||||
| jq -r '.secret.secretValue'
|
||||
}
|
||||
|
||||
for name in GCP_CREDENTIAL TOKEN_GITHUB_CICD DIGITALOCEAN_ACCESS_TOKEN; do
|
||||
val="$(fetch_secret "$name" 2>/dev/null || echo "")"
|
||||
[ -n "$val" ] && [ "$val" != "null" ] || continue
|
||||
echo "::add-mask::${val}"
|
||||
{ echo "${name}<<ENDOFSECRET"; echo "${val}"; echo "ENDOFSECRET"; } >> "$GITHUB_OUTPUT"
|
||||
done
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.resolve.outputs.environment }}
|
||||
token: ${{ steps.kms.outputs.TOKEN_GITHUB_CICD }}
|
||||
|
||||
- name: Update manifest
|
||||
run: |
|
||||
bash scripts/ci/update-manifest.sh \
|
||||
"${{ needs.resolve.outputs.service_key }}" \
|
||||
"${{ needs.resolve.outputs.service_sha }}" \
|
||||
"${{ needs.resolve.outputs.service_image }}"
|
||||
|
||||
- name: Create deploy PR (triggers E2E before merge)
|
||||
env:
|
||||
GH_TOKEN: ${{ steps.kms.outputs.TOKEN_GITHUB_CICD }}
|
||||
run: |
|
||||
git config user.name "zoo-ci"
|
||||
git config user.email "[email protected]"
|
||||
git add manifests/
|
||||
if git diff --cached --quiet; then
|
||||
echo "No manifest changes"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
SERVICE="${{ needs.resolve.outputs.service_key }}"
|
||||
SHA="${{ needs.resolve.outputs.service_sha }}"
|
||||
VERSION="${{ github.event.client_payload.version }}"
|
||||
LABEL="${VERSION:-${SHA::7}}"
|
||||
BRANCH="deploy/${SERVICE}-${LABEL}"
|
||||
BASE="${{ needs.resolve.outputs.environment }}"
|
||||
|
||||
git checkout -b "${BRANCH}"
|
||||
git commit -m "deploy: ${SERVICE} v${LABEL}"
|
||||
git push origin "${BRANCH}"
|
||||
|
||||
gh pr create \
|
||||
--base "${BASE}" \
|
||||
--head "${BRANCH}" \
|
||||
--title "deploy: ${SERVICE} v${LABEL}" \
|
||||
--body "$(cat <<EOF
|
||||
**Service**: \`${SERVICE}\`
|
||||
**Version**: \`${LABEL}\`
|
||||
**Image**: \`${{ needs.resolve.outputs.service_image }}\`
|
||||
|
||||
PR E2E runs automatically. Auto-merges on pass (minor/patch).
|
||||
EOF
|
||||
)"
|
||||
|
||||
gh pr merge "${BRANCH}" --auto --squash 2>/dev/null || true
|
||||
|
||||
deploy-single:
|
||||
name: Deploy ${{ needs.resolve.outputs.service_key }}
|
||||
runs-on: [zoo-build-linux-amd64]
|
||||
needs: [resolve]
|
||||
if: needs.resolve.outputs.mode == 'deploy-single' && github.event_name == 'push'
|
||||
environment: ${{ needs.resolve.outputs.environment }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.resolve.outputs.environment }}
|
||||
|
||||
- name: Fetch secrets from KMS
|
||||
id: kms
|
||||
env:
|
||||
KMS_CLIENT_ID: ${{ secrets.KMS_CLIENT_ID }}
|
||||
KMS_CLIENT_SECRET: ${{ secrets.KMS_CLIENT_SECRET }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
KMS_URL="${KMS_URL:-https://kms.hanzo.ai}"
|
||||
WORKSPACE="zoo-platform"
|
||||
|
||||
ACCESS_TOKEN="$(
|
||||
curl -fsS --max-time 10 -X POST "${KMS_URL}/api/v1/auth/universal-auth/login" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"clientId\":\"${KMS_CLIENT_ID}\",\"clientSecret\":\"${KMS_CLIENT_SECRET}\"}" \
|
||||
| jq -r '.accessToken'
|
||||
)"
|
||||
|
||||
[ -n "${ACCESS_TOKEN}" ] && [ "${ACCESS_TOKEN}" != "null" ] || { echo "::error::KMS auth failed"; exit 1; }
|
||||
|
||||
fetch_secret() {
|
||||
curl -fsS "${KMS_URL}/api/v3/secrets/raw/${1}?workspaceSlug=${WORKSPACE}&environment=prod&secretPath=/ci&viewSecretValue=true&include_imports=true" \
|
||||
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
|
||||
| jq -r '.secret.secretValue'
|
||||
}
|
||||
|
||||
for name in GCP_CREDENTIAL TOKEN_GITHUB_CICD DIGITALOCEAN_ACCESS_TOKEN; do
|
||||
val="$(fetch_secret "$name" 2>/dev/null || echo "")"
|
||||
[ -n "$val" ] && [ "$val" != "null" ] || continue
|
||||
echo "::add-mask::${val}"
|
||||
{ echo "${name}<<ENDOFSECRET"; echo "${val}"; echo "ENDOFSECRET"; } >> "$GITHUB_OUTPUT"
|
||||
done
|
||||
|
||||
- name: Install doctl
|
||||
uses: digitalocean/action-doctl@v2
|
||||
with:
|
||||
token: ${{ steps.kms.outputs.DIGITALOCEAN_ACCESS_TOKEN }}
|
||||
|
||||
- name: Configure kubectl
|
||||
run: doctl kubernetes cluster kubeconfig save "${{ env.CLUSTER }}"
|
||||
|
||||
- name: Apply kustomization
|
||||
run: |
|
||||
NS="${{ needs.resolve.outputs.namespace }}"
|
||||
kubectl apply -k "k8s/${NS}"
|
||||
|
||||
- name: Wait for rollout
|
||||
run: |
|
||||
SERVICE="${{ needs.resolve.outputs.service_key }}"
|
||||
NS="${{ needs.resolve.outputs.namespace }}"
|
||||
|
||||
for i in $(seq 1 30); do
|
||||
READY=$(kubectl get deploy,sts -n "${NS}" -l app="${SERVICE}" \
|
||||
-o jsonpath='{.items[0].status.readyReplicas}' 2>/dev/null || echo "0")
|
||||
DESIRED=$(kubectl get deploy,sts -n "${NS}" -l app="${SERVICE}" \
|
||||
-o jsonpath='{.items[0].spec.replicas}' 2>/dev/null || echo "1")
|
||||
if [ "${READY}" = "${DESIRED}" ] && [ "${READY}" != "0" ]; then
|
||||
echo "${SERVICE}: ${READY}/${DESIRED} ready"
|
||||
break
|
||||
fi
|
||||
echo "${SERVICE}: ${READY:-0}/${DESIRED:-?} ready (attempt $i/30)"
|
||||
sleep 10
|
||||
done
|
||||
|
||||
# ============================================================
|
||||
# Mode: deploy-all -- deploy all services from manifest
|
||||
# ============================================================
|
||||
deploy-all:
|
||||
name: Deploy All Services
|
||||
runs-on: [zoo-build-linux-amd64]
|
||||
needs: [resolve]
|
||||
if: needs.resolve.outputs.mode == 'deploy-all'
|
||||
environment: ${{ needs.resolve.outputs.environment }}
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.resolve.outputs.environment }}
|
||||
|
||||
- name: Fetch secrets from KMS
|
||||
id: kms
|
||||
env:
|
||||
KMS_CLIENT_ID: ${{ secrets.KMS_CLIENT_ID }}
|
||||
KMS_CLIENT_SECRET: ${{ secrets.KMS_CLIENT_SECRET }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
KMS_URL="${KMS_URL:-https://kms.hanzo.ai}"
|
||||
WORKSPACE="zoo-platform"
|
||||
|
||||
ACCESS_TOKEN="$(
|
||||
curl -fsS --max-time 10 -X POST "${KMS_URL}/api/v1/auth/universal-auth/login" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"clientId\":\"${KMS_CLIENT_ID}\",\"clientSecret\":\"${KMS_CLIENT_SECRET}\"}" \
|
||||
| jq -r '.accessToken'
|
||||
)"
|
||||
|
||||
[ -n "${ACCESS_TOKEN}" ] && [ "${ACCESS_TOKEN}" != "null" ] || { echo "::error::KMS auth failed"; exit 1; }
|
||||
|
||||
fetch_secret() {
|
||||
curl -fsS "${KMS_URL}/api/v3/secrets/raw/${1}?workspaceSlug=${WORKSPACE}&environment=prod&secretPath=/ci&viewSecretValue=true&include_imports=true" \
|
||||
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
|
||||
| jq -r '.secret.secretValue'
|
||||
}
|
||||
|
||||
for name in GCP_CREDENTIAL TOKEN_GITHUB_CICD DIGITALOCEAN_ACCESS_TOKEN; do
|
||||
val="$(fetch_secret "$name" 2>/dev/null || echo "")"
|
||||
[ -n "$val" ] && [ "$val" != "null" ] || continue
|
||||
echo "::add-mask::${val}"
|
||||
{ echo "${name}<<ENDOFSECRET"; echo "${val}"; echo "ENDOFSECRET"; } >> "$GITHUB_OUTPUT"
|
||||
done
|
||||
|
||||
- name: Install doctl
|
||||
uses: digitalocean/action-doctl@v2
|
||||
with:
|
||||
token: ${{ steps.kms.outputs.DIGITALOCEAN_ACCESS_TOKEN }}
|
||||
|
||||
- name: Configure kubectl
|
||||
run: doctl kubernetes cluster kubeconfig save "${{ env.CLUSTER }}"
|
||||
|
||||
- name: Deploy all from manifest
|
||||
run: bash scripts/ci/deploy-all.sh "${{ needs.resolve.outputs.environment }}"
|
||||
env:
|
||||
DEPLOY_NAMESPACE: ${{ needs.resolve.outputs.namespace }}
|
||||
|
||||
# ============================================================
|
||||
# E2E tests
|
||||
# ============================================================
|
||||
e2e:
|
||||
name: E2E
|
||||
runs-on: [zoo-build-linux-amd64]
|
||||
needs: [resolve, deploy-all]
|
||||
if: always() && (needs.resolve.outputs.mode == 'deploy-all' || needs.resolve.outputs.mode == 'e2e-only') && needs.resolve.result == 'success'
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.resolve.outputs.environment }}
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 20
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci 2>/dev/null || true
|
||||
|
||||
- name: Install Playwright
|
||||
run: npx playwright install --with-deps chromium 2>/dev/null || true
|
||||
|
||||
- name: Fetch secrets from KMS
|
||||
id: kms
|
||||
env:
|
||||
KMS_CLIENT_ID: ${{ secrets.KMS_CLIENT_ID }}
|
||||
KMS_CLIENT_SECRET: ${{ secrets.KMS_CLIENT_SECRET }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
KMS_URL="${KMS_URL:-https://kms.hanzo.ai}"
|
||||
WORKSPACE="zoo-platform"
|
||||
|
||||
ACCESS_TOKEN="$(
|
||||
curl -fsS --max-time 10 -X POST "${KMS_URL}/api/v1/auth/universal-auth/login" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "{\"clientId\":\"${KMS_CLIENT_ID}\",\"clientSecret\":\"${KMS_CLIENT_SECRET}\"}" \
|
||||
| jq -r '.accessToken'
|
||||
)"
|
||||
|
||||
[ -n "${ACCESS_TOKEN}" ] && [ "${ACCESS_TOKEN}" != "null" ] || { echo "::error::KMS auth failed"; exit 1; }
|
||||
|
||||
fetch_secret() {
|
||||
curl -fsS "${KMS_URL}/api/v3/secrets/raw/${1}?workspaceSlug=${WORKSPACE}&environment=prod&secretPath=/ci&viewSecretValue=true&include_imports=true" \
|
||||
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
|
||||
| jq -r '.secret.secretValue'
|
||||
}
|
||||
|
||||
for name in GCP_CREDENTIAL TOKEN_GITHUB_CICD DIGITALOCEAN_ACCESS_TOKEN; do
|
||||
val="$(fetch_secret "$name" 2>/dev/null || echo "")"
|
||||
[ -n "$val" ] && [ "$val" != "null" ] || continue
|
||||
echo "::add-mask::${val}"
|
||||
{ echo "${name}<<ENDOFSECRET"; echo "${val}"; echo "ENDOFSECRET"; } >> "$GITHUB_OUTPUT"
|
||||
done
|
||||
|
||||
- name: Install doctl
|
||||
uses: digitalocean/action-doctl@v2
|
||||
with:
|
||||
token: ${{ steps.kms.outputs.DIGITALOCEAN_ACCESS_TOKEN }}
|
||||
|
||||
- name: Configure kubectl
|
||||
run: doctl kubernetes cluster kubeconfig save "${{ env.CLUSTER }}"
|
||||
|
||||
- name: Run E2E
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
E2E_ENV: ${{ needs.resolve.outputs.environment }}
|
||||
E2E_NAMESPACE: ${{ needs.resolve.outputs.namespace }}
|
||||
CI: "true"
|
||||
run: |
|
||||
if [ -f e2e/run.sh ]; then
|
||||
bash e2e/run.sh
|
||||
elif [ -f playwright.config.ts ]; then
|
||||
npx playwright test
|
||||
else
|
||||
echo "No E2E tests configured yet -- skipping"
|
||||
fi
|
||||
|
||||
- name: Upload results
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: e2e-${{ needs.resolve.outputs.environment }}-${{ github.run_number }}
|
||||
path: |
|
||||
playwright-report/
|
||||
e2e/test-results/
|
||||
retention-days: 7
|
||||
|
||||
- name: Collect failure diagnostics
|
||||
if: failure()
|
||||
run: |
|
||||
NS="${{ needs.resolve.outputs.namespace }}"
|
||||
echo "=== Pod Status ==="
|
||||
kubectl get pods -n "${NS}" -o wide 2>/dev/null || true
|
||||
echo ""
|
||||
echo "=== Events (last 20) ==="
|
||||
kubectl get events -n "${NS}" --sort-by='.lastTimestamp' 2>/dev/null | tail -20 || true
|
||||
echo ""
|
||||
echo "=== CrashLooping / Error Pod Logs ==="
|
||||
for pod in $(kubectl get pods -n "${NS}" --no-headers 2>/dev/null | grep -vE 'Running|Completed' | awk '{print $1}'); do
|
||||
echo "--- ${pod} ---"
|
||||
kubectl logs -n "${NS}" "${pod}" --tail=50 2>/dev/null || true
|
||||
done
|
||||
@@ -37,7 +37,7 @@ spec:
|
||||
sources:
|
||||
- repoURL: oci.hanzo.ai/charts
|
||||
chart: app
|
||||
targetRevision: 0.1.0
|
||||
targetRevision: 0.1.11
|
||||
helm:
|
||||
releaseName: '{{ .path.filename | trimSuffix ".yaml" }}'
|
||||
valueFiles:
|
||||
|
||||
Reference in New Issue
Block a user