mirror of
https://github.com/hanzoai/ingress.git
synced 2026-10-26 02:47:15 +00:00
edgeSecrets ran on every boot, so an edge that orders no certificates still had to reach KMS before it served anything — including routes whose certificates come from a Kubernetes Secret, and plain-HTTP routes. It now returns the identity seal when no resolver orders certificates, and reaches KMS only when there is ACME state to seal. Adoption covers ONE store. It admits the first unsealed document and the same document again — a store is read more than once before the write that seals it lands, and a replica that cannot write reads it every poll — but not a different one. A seal left adopting was taking whatever plaintext it was handed for as long as the process ran. The setting's comment now says what it does: it writes, it covers one store, and it adopts again after a restart, which is why it is removed once it has done its work. One spelling for the KMS credential. The bare IAM_CLIENT_ID fallback undid the reason the names are prefixed — a pod carrying another subsystem's IAM client had it picked up here. k8s/hanzo/deployment.yaml and service.yaml are deleted. They described a hostNetwork DaemonSet named hanzo-ingress binding hostPort 80 and 443, while what runs in that cluster is a Deployment named ingress declared in universe. `kubectl apply -f k8s/hanzo/` appears in this README and in the sibling gateway's Makefile, so the file was one command from taking :80 and :443 on every node beside the workload that already had them. Universe is the one place that says what runs; the cluster-level objects stay here. The OTLP endpoint paths in the docs and the vendored Knative fixture are restored: /v1/metrics is the OpenTelemetry receiver path, not a route of ours.
199 lines
6.7 KiB
Go
199 lines
6.7 KiB
Go
// Package kms reads this deployment's secrets from Hanzo KMS.
|
|
//
|
|
// The edge holds two things an attacker wants more than any request it proxies:
|
|
// the ACME account key, which can re-issue a certificate for every domain the
|
|
// estate serves, and the DNS credential the ACME challenge is solved with. Both
|
|
// used to arrive as a base64 field in a Kubernetes Secret and live in the pod's
|
|
// environment or in a file on the node. This package is where they come from
|
|
// now: fetched once at startup, held in memory, never written down.
|
|
//
|
|
// It speaks the luxfi/kms HTTP contract — the same two calls the gateway's
|
|
// routes loader makes, so the estate has one KMS conversation rather than two:
|
|
//
|
|
// POST /v1/kms/auth/login {clientId, clientSecret} -> {accessToken}
|
|
// GET /v1/kms/secrets/{path}/{name}?env= -> {"secret":{"value"}}
|
|
//
|
|
// The org is the TOKEN's, never a path segment: the bearer minted at login
|
|
// carries owner=<org> and the server scopes the read to it. Which credential
|
|
// logs in is the one honest place an org belongs.
|
|
package kms
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// Env names. The prefix is this service's, and it is the ONLY spelling: a pod
|
|
// that also carries another subsystem's KMS credentials cannot have them picked
|
|
// up here by accident, which a bare fallback spelling would undo.
|
|
const (
|
|
envEndpoint = "INGRESS_KMS_ENDPOINT"
|
|
envClientID = "INGRESS_KMS_CLIENT_ID"
|
|
envSecret = "INGRESS_KMS_CLIENT_SECRET"
|
|
envOrg = "INGRESS_KMS_ORG"
|
|
envEnv = "INGRESS_KMS_ENV"
|
|
envPath = "INGRESS_KMS_PATH"
|
|
)
|
|
|
|
// Names of the secrets this service reads, under the configured path. They are
|
|
// constants rather than env vars: which secrets the ingress needs is a property
|
|
// of the ingress, not of the environment it runs in.
|
|
const (
|
|
// AccountSeal is the 32-byte key the ACME state is sealed under, hex or
|
|
// standard base64.
|
|
AccountSeal = "acme-seal"
|
|
// DNSToken is the Cloudflare API token the DNS-01 challenge is solved with.
|
|
// It is a ZONE-scoped DNS:Edit token, not the account-global API key.
|
|
DNSToken = "cloudflare-token"
|
|
)
|
|
|
|
// Client reads secrets for one org, at one path, in one environment.
|
|
type Client struct {
|
|
endpoint string
|
|
clientID string
|
|
secret string
|
|
org string
|
|
env string
|
|
path string
|
|
http *http.Client
|
|
}
|
|
|
|
// FromEnv builds the client this deployment is configured for. It returns
|
|
// (nil, nil) when INGRESS_KMS_ENDPOINT is unset — that deployment reads no
|
|
// secrets from KMS and the caller decides what that means — and an error when
|
|
// the endpoint is set but the credentials to use it are not, which is a
|
|
// half-configured deployment rather than an unconfigured one.
|
|
func FromEnv() (*Client, error) {
|
|
endpoint := strings.TrimRight(strings.TrimSpace(os.Getenv(envEndpoint)), "/")
|
|
if endpoint == "" {
|
|
return nil, nil
|
|
}
|
|
// The credential travels in the body of the login and the bearer travels
|
|
// in a header of every read, so the endpoint has to be one that encrypts
|
|
// them. A plaintext endpoint is a configuration to fix, not a degraded mode
|
|
// to run in.
|
|
if !strings.HasPrefix(endpoint, "https://") {
|
|
return nil, fmt.Errorf("kms: %s is %q; the endpoint carries a client secret and must be https", envEndpoint, endpoint)
|
|
}
|
|
id := strings.TrimSpace(os.Getenv(envClientID))
|
|
secret := strings.TrimSpace(os.Getenv(envSecret))
|
|
if id == "" || secret == "" {
|
|
return nil, fmt.Errorf("kms: %s is set but %s/%s are empty", envEndpoint, envClientID, envSecret)
|
|
}
|
|
return &Client{
|
|
endpoint: endpoint,
|
|
clientID: id,
|
|
secret: secret,
|
|
org: firstOf(os.Getenv(envOrg), "hanzo"),
|
|
env: firstOf(os.Getenv(envEnv), "default"),
|
|
path: firstOf(os.Getenv(envPath), "ingress"),
|
|
http: &http.Client{Timeout: 10 * time.Second},
|
|
}, nil
|
|
}
|
|
|
|
// Org reports which org's secrets this client reads. Logged at startup so a
|
|
// deployment that authenticated as the wrong tenant says so before it serves.
|
|
func (c *Client) Org() string { return c.org }
|
|
|
|
// Get returns the value of one secret under this client's path.
|
|
func (c *Client) Get(ctx context.Context, name string) (string, error) {
|
|
token, err := c.login(ctx)
|
|
if err != nil {
|
|
return "", fmt.Errorf("kms: auth: %w", err)
|
|
}
|
|
// Each segment is escaped on its own: escaping the joined string would
|
|
// encode the separators away and the server would read one long name.
|
|
segs := strings.Split(c.path+"/"+name, "/")
|
|
for i, s := range segs {
|
|
segs[i] = url.PathEscape(s)
|
|
}
|
|
u := fmt.Sprintf("%s/v1/kms/secrets/%s?env=%s",
|
|
c.endpoint, strings.Join(segs, "/"), url.QueryEscape(c.env))
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
req.Header.Set("Authorization", "Bearer "+token)
|
|
|
|
resp, err := c.http.Do(req)
|
|
if err != nil {
|
|
return "", fmt.Errorf("kms: get %s: %w", name, err)
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
// The body can carry the server's own error text but never the secret,
|
|
// so it is safe to quote and it is the only way to tell a missing path
|
|
// from a scope the token does not hold.
|
|
body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
|
return "", fmt.Errorf("kms: get %s returned %d: %s", name, resp.StatusCode, bytes.TrimSpace(body))
|
|
}
|
|
var out struct {
|
|
Secret struct {
|
|
Value string `json:"value"`
|
|
} `json:"secret"`
|
|
}
|
|
if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<20)).Decode(&out); err != nil {
|
|
return "", fmt.Errorf("kms: decode %s: %w", name, err)
|
|
}
|
|
if out.Secret.Value == "" {
|
|
return "", fmt.Errorf("kms: %s/%s is empty", c.path, name)
|
|
}
|
|
return out.Secret.Value, nil
|
|
}
|
|
|
|
func (c *Client) login(ctx context.Context) (string, error) {
|
|
payload, err := json.Marshal(map[string]string{
|
|
"clientId": c.clientID,
|
|
"clientSecret": c.secret,
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
|
c.endpoint+"/v1/kms/auth/login", bytes.NewReader(payload))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
|
|
resp, err := c.http.Do(req)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != http.StatusOK {
|
|
// Deliberately no body here: a login response can echo the credential
|
|
// that failed, and this error is going to a log.
|
|
return "", fmt.Errorf("login returned %d", resp.StatusCode)
|
|
}
|
|
var out struct {
|
|
AccessToken string `json:"accessToken"`
|
|
}
|
|
if err := json.NewDecoder(io.LimitReader(resp.Body, 1<<16)).Decode(&out); err != nil {
|
|
return "", err
|
|
}
|
|
if out.AccessToken == "" {
|
|
return "", errors.New("login returned no access token")
|
|
}
|
|
return out.AccessToken, nil
|
|
}
|
|
|
|
func firstOf(values ...string) string {
|
|
for _, v := range values {
|
|
if v = strings.TrimSpace(v); v != "" {
|
|
return v
|
|
}
|
|
}
|
|
return ""
|
|
}
|