feat: replicate sidecar patches for SQLite replication to S3
ConfigMap, S3 storage service, bridge deployment patch, apply script. Same hanzoai/replicate image as <tenant>. DigitalOcean Spaces backend.
This commit is contained in:
Hanzo AI
@@ -0,0 +1,103 @@
|
||||
# Replicate sidecar for zoo-bridge Deployment
|
||||
# Bridge currently uses PostgreSQL but is migrating to Base/SQLite.
|
||||
# This patch adds:
|
||||
# 1. A PVC for bridge data
|
||||
# 2. init container: restore SQLite from S3 on cold start
|
||||
# 3. sidecar: continuous WAL replication to S3
|
||||
# 4. data volume mount on the bridge container
|
||||
#
|
||||
# IMPORTANT: When using SQLite, replicas must be set to 1.
|
||||
# SQLite does not support concurrent writers from multiple pods.
|
||||
#
|
||||
# Apply with:
|
||||
# kubectl apply -f k8s/patches/replicate-bridge.yaml
|
||||
# kubectl -n zoo rollout restart deploy/zoo-bridge
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: zoo-bridge-data
|
||||
namespace: zoo
|
||||
spec:
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
storageClassName: do-block-storage
|
||||
resources:
|
||||
requests:
|
||||
storage: 2Gi
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: zoo-bridge
|
||||
namespace: zoo
|
||||
spec:
|
||||
replicas: 1
|
||||
template:
|
||||
spec:
|
||||
initContainers:
|
||||
- name: replicate-restore
|
||||
image: ghcr.io/hanzoai/replicate:latest
|
||||
args: ["restore", "-if-db-not-exists", "-if-replica-exists", "/app/data/data.db"]
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /app/data
|
||||
- name: replicate-config
|
||||
mountPath: /etc/litestream.yml
|
||||
subPath: litestream.yml
|
||||
env:
|
||||
- name: LITESTREAM_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: s3-credentials
|
||||
key: ACCESS_KEY
|
||||
- name: LITESTREAM_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: s3-credentials
|
||||
key: SECRET_KEY
|
||||
- name: POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
containers:
|
||||
- name: bridge
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /app/data
|
||||
- name: replicate
|
||||
image: ghcr.io/hanzoai/replicate:latest
|
||||
args: ["replicate"]
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /app/data
|
||||
- name: replicate-config
|
||||
mountPath: /etc/litestream.yml
|
||||
subPath: litestream.yml
|
||||
env:
|
||||
- name: LITESTREAM_ACCESS_KEY_ID
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: s3-credentials
|
||||
key: ACCESS_KEY
|
||||
- name: LITESTREAM_SECRET_ACCESS_KEY
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: s3-credentials
|
||||
key: SECRET_KEY
|
||||
- name: POD_NAME
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: metadata.name
|
||||
resources:
|
||||
requests:
|
||||
memory: 16Mi
|
||||
cpu: 10m
|
||||
limits:
|
||||
memory: 64Mi
|
||||
volumes:
|
||||
- name: data
|
||||
persistentVolumeClaim:
|
||||
claimName: zoo-bridge-data
|
||||
- name: replicate-config
|
||||
configMap:
|
||||
name: replicate-config
|
||||
@@ -0,0 +1,25 @@
|
||||
# Replicate ConfigMap — continuous encrypted SQLite replication to S3
|
||||
# Shared by all Zoo services that use Base/SQLite.
|
||||
# Applied separately from service patches so it exists before any sidecar starts.
|
||||
#
|
||||
# Each pod's replica path is namespaced by POD_NAME (injected via fieldRef),
|
||||
# so zoo-bridge-0 and zoo-bridge-1 replicate to distinct S3 prefixes.
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: replicate-config
|
||||
namespace: zoo
|
||||
data:
|
||||
litestream.yml: |
|
||||
dbs:
|
||||
- path: /app/data/data.db
|
||||
replicas:
|
||||
- type: s3
|
||||
bucket: zoo-data
|
||||
path: ${POD_NAME}
|
||||
endpoint: ${LITESTREAM_S3_ENDPOINT:-http://s3.zoo.svc.cluster.local:9000}
|
||||
region: sfo3
|
||||
force-path-style: true
|
||||
sync-interval: 1s
|
||||
snapshot-interval: 1h
|
||||
@@ -0,0 +1,98 @@
|
||||
# S3-compatible object storage for Zoo replicate sidecars
|
||||
# Uses hanzoai/s3 (MinIO). Backed by DigitalOcean block storage.
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: s3
|
||||
namespace: zoo
|
||||
labels:
|
||||
app: s3
|
||||
app.kubernetes.io/name: s3
|
||||
app.kubernetes.io/part-of: zoo
|
||||
spec:
|
||||
ports:
|
||||
- port: 9000
|
||||
targetPort: 9000
|
||||
name: s3
|
||||
selector:
|
||||
app: s3
|
||||
clusterIP: None
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: StatefulSet
|
||||
metadata:
|
||||
name: s3
|
||||
namespace: zoo
|
||||
labels:
|
||||
app: s3
|
||||
app.kubernetes.io/name: s3
|
||||
app.kubernetes.io/part-of: zoo
|
||||
spec:
|
||||
serviceName: s3
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: s3
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: s3
|
||||
app.kubernetes.io/name: s3
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: ghcr-secret
|
||||
- name: hanzo
|
||||
containers:
|
||||
- name: s3
|
||||
image: ghcr.io/hanzoai/s3:latest
|
||||
imagePullPolicy: Always
|
||||
ports:
|
||||
- containerPort: 9000
|
||||
name: s3
|
||||
env:
|
||||
- name: MINIO_ROOT_USER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: s3-credentials
|
||||
key: ACCESS_KEY
|
||||
- name: MINIO_ROOT_PASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: s3-credentials
|
||||
key: SECRET_KEY
|
||||
args: ["server", "/data"]
|
||||
volumeMounts:
|
||||
- name: s3-data
|
||||
mountPath: /data
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /minio/health/ready
|
||||
port: 9000
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /minio/health/live
|
||||
port: 9000
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 30
|
||||
persistentVolumeClaimRetentionPolicy:
|
||||
whenDeleted: Retain
|
||||
whenScaled: Retain
|
||||
volumeClaimTemplates:
|
||||
- metadata:
|
||||
name: s3-data
|
||||
spec:
|
||||
accessModes: ["ReadWriteOnce"]
|
||||
storageClassName: do-block-storage
|
||||
resources:
|
||||
requests:
|
||||
storage: 20Gi
|
||||
Executable
+60
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env bash
|
||||
# Apply Replicate sidecars for SQLite replication to S3.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/apply-replicate.sh # zoo-devnet (default)
|
||||
# ./scripts/apply-replicate.sh do-sfo3-zoo-k8s # specify context
|
||||
#
|
||||
# Prerequisites:
|
||||
# - ghcr.io/hanzoai/replicate:latest accessible (ghcr-secret)
|
||||
# - s3-credentials secret exists in target namespace
|
||||
# - replicate-config ConfigMap applied first (done by this script)
|
||||
#
|
||||
# What this does:
|
||||
# 1. Deploys S3 storage service (hanzoai/s3)
|
||||
# 2. Applies the shared replicate-config ConfigMap
|
||||
# 3. Ensures s3-credentials secret exists (errors if missing)
|
||||
# 4. Patches zoo-bridge (Deployment) with replicate sidecar + PVC
|
||||
# 5. Restarts patched workloads
|
||||
set -euo pipefail
|
||||
|
||||
CTX="${1:-do-sfo3-zoo-k8s}"
|
||||
DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
NS="${2:-zoo-devnet}"
|
||||
|
||||
echo "==> Context: $CTX"
|
||||
echo "==> Namespace: $NS"
|
||||
|
||||
# Check s3-credentials exist
|
||||
if ! kubectl --context "$CTX" -n "$NS" get secret s3-credentials &>/dev/null; then
|
||||
echo "ERROR: s3-credentials secret not found in $NS namespace."
|
||||
echo "Create it first:"
|
||||
echo " kubectl --context $CTX -n $NS create secret generic s3-credentials \\"
|
||||
echo " --from-literal=ACCESS_KEY=<key> --from-literal=SECRET_KEY=<secret>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "==> Deploying S3 storage service"
|
||||
kubectl --context "$CTX" apply -f "$DIR/k8s/patches/s3.yaml"
|
||||
|
||||
echo "==> Waiting for S3 readiness"
|
||||
kubectl --context "$CTX" -n "$NS" rollout status statefulset/s3 --timeout=120s
|
||||
|
||||
echo "==> Applying replicate-config ConfigMap"
|
||||
kubectl --context "$CTX" apply -f "$DIR/k8s/patches/replicate-config.yaml"
|
||||
|
||||
echo "==> Patching zoo-bridge (Deployment)"
|
||||
kubectl --context "$CTX" apply -f "$DIR/k8s/patches/replicate-bridge.yaml"
|
||||
|
||||
echo "==> Restarting patched workloads"
|
||||
kubectl --context "$CTX" -n "$NS" rollout restart deploy/zoo-bridge
|
||||
|
||||
echo "==> Waiting for rollouts"
|
||||
for w in deploy/zoo-bridge; do
|
||||
echo " $w..."
|
||||
kubectl --context "$CTX" -n "$NS" rollout status "$w" --timeout=120s
|
||||
done
|
||||
|
||||
echo "==> Done. Replicate sidecars active on: zoo-bridge"
|
||||
echo " S3 endpoint: http://s3.$NS.svc.cluster.local:9000"
|
||||
echo " Bucket: zoo-data"
|
||||
Reference in New Issue
Block a user