mirror of
https://github.com/luxfi/lux-gpu-ci.git
synced 2026-10-30 16:20:42 +00:00
zeekay
dde006617a
EKS cluster with Karpenter v1 NodePool that scales g4dn.xlarge spot instances 0..1 on demand for CUDA workflows. ARC RunnerScaleSet registers as [self-hosted, linux, amd64, cuda, gpu, t4]. Components: - terraform/ — VPC, EKS 1.31, system node group, Karpenter IAM/SQS - helm/karpenter-nodepool.yaml — v1 NodePool + EC2NodeClass for GPU spot - helm/nvidia-device-plugin-values.yaml — DS with GPU taint toleration - helm/arc-values.yaml + helm/runner-scaleset.yaml — ARC v0.9.x - bootstrap.sh — one-shot deploy - Makefile — init/apply/destroy/verify/cost-estimate Cost: ~$138/mo idle (EKS control plane + tiny system node + NAT), ~$0.027/build for a 10-minute g4dn.xlarge spot run.
85 lines
2.3 KiB
YAML
85 lines
2.3 KiB
YAML
###############################################################################
|
|
# Karpenter v1 NodePool + EC2NodeClass for GPU spot instances.
|
|
#
|
|
# Behavior:
|
|
# - 0 GPU nodes idle. When a pod with `nvidia.com/gpu` request and matching
|
|
# toleration is unschedulable, Karpenter provisions a g4dn.xlarge spot.
|
|
# - Falls back to g5.xlarge if g4dn unavailable.
|
|
# - 30s after the pod terminates, Karpenter consolidates the node away.
|
|
# - amazon-linux-2023 AMI ships GPU drivers via NVIDIA device plugin DS.
|
|
###############################################################################
|
|
---
|
|
apiVersion: karpenter.k8s.aws/v1
|
|
kind: EC2NodeClass
|
|
metadata:
|
|
name: gpu
|
|
spec:
|
|
amiSelectorTerms:
|
|
- alias: al2023@latest
|
|
role: ${KARPENTER_NODE_IAM_ROLE_NAME}
|
|
subnetSelectorTerms:
|
|
- tags:
|
|
karpenter.sh/discovery: ${CLUSTER_NAME}
|
|
securityGroupSelectorTerms:
|
|
- tags:
|
|
karpenter.sh/discovery: ${CLUSTER_NAME}
|
|
blockDeviceMappings:
|
|
- deviceName: /dev/xvda
|
|
ebs:
|
|
volumeSize: 100Gi
|
|
volumeType: gp3
|
|
iops: 3000
|
|
throughput: 125
|
|
deleteOnTermination: true
|
|
encrypted: true
|
|
metadataOptions:
|
|
httpEndpoint: enabled
|
|
httpPutResponseHopLimit: 2
|
|
httpTokens: required
|
|
tags:
|
|
Project: lux-gpu-ci
|
|
Component: gpu-runner
|
|
---
|
|
apiVersion: karpenter.sh/v1
|
|
kind: NodePool
|
|
metadata:
|
|
name: gpu-spot
|
|
spec:
|
|
template:
|
|
metadata:
|
|
labels:
|
|
workload: gpu-ci
|
|
nvidia.com/gpu.present: "true"
|
|
spec:
|
|
nodeClassRef:
|
|
group: karpenter.k8s.aws
|
|
kind: EC2NodeClass
|
|
name: gpu
|
|
requirements:
|
|
- key: kubernetes.io/arch
|
|
operator: In
|
|
values: ["amd64"]
|
|
- key: kubernetes.io/os
|
|
operator: In
|
|
values: ["linux"]
|
|
- key: node.kubernetes.io/instance-type
|
|
operator: In
|
|
values: ["g4dn.xlarge", "g5.xlarge"]
|
|
- key: karpenter.sh/capacity-type
|
|
operator: In
|
|
values: ["spot"]
|
|
taints:
|
|
- key: nvidia.com/gpu
|
|
value: "present"
|
|
effect: NoSchedule
|
|
expireAfter: 24h
|
|
terminationGracePeriod: 5m
|
|
limits:
|
|
cpu: "8"
|
|
memory: 32Gi
|
|
"nvidia.com/gpu": 1
|
|
disruption:
|
|
consolidationPolicy: WhenEmpty
|
|
consolidateAfter: 30s
|
|
weight: 10
|