mirror of
https://github.com/hanzo-ml/trainer.git
synced 2026-09-24 00:31:34 +00:00
Tariq HasanandGitHub
1f57df4484
* ci(boilerplate): enforce copyright headers on YAML files Signed-off-by: tariq-hasan <[email protected]> * ci(boilerplate): enforce year-less header for new files Signed-off-by: tariq-hasan <[email protected]> --------- Signed-off-by: tariq-hasan <[email protected]>
83 lines
2.4 KiB
YAML
83 lines
2.4 KiB
YAML
# Copyright The Kubeflow Authors.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# Advanced Example 2: Kueue Integration
|
|
#
|
|
# Queue-based scheduling and quota management with Kueue. The TrainJob is
|
|
# created suspended; Kueue unsuspends it when its admission check passes
|
|
# and the LocalQueue/ClusterQueue has capacity.
|
|
#
|
|
# Prerequisites:
|
|
# - Kueue installed in the cluster
|
|
# - A LocalQueue named "training-queue" in the target namespace, backed
|
|
# by a ClusterQueue with quota
|
|
#
|
|
# Apply: kubectl apply -f 03-kueue-integration.yaml
|
|
# Check: kubectl get trainjob kueue-example
|
|
# Queue: kubectl get workload -l kueue.x-k8s.io/job-name=kueue-example
|
|
# Pods: kubectl get pods -l jobset.sigs.k8s.io/jobset-name=kueue-example
|
|
# Clean: kubectl delete trainjob kueue-example
|
|
|
|
apiVersion: trainer.kubeflow.org/v1alpha1
|
|
kind: TrainJob
|
|
metadata:
|
|
name: kueue-example
|
|
labels:
|
|
# Tells Kueue which LocalQueue to admit this TrainJob into.
|
|
kueue.x-k8s.io/queue-name: training-queue
|
|
spec:
|
|
runtimeRef:
|
|
apiGroup: trainer.kubeflow.org
|
|
kind: ClusterTrainingRuntime
|
|
name: torch-distributed
|
|
|
|
# Required: Kueue admits the workload by flipping suspend to false.
|
|
suspend: true
|
|
|
|
# Optional: set to "kueue.x-k8s.io/multikueue" to delegate to MultiKueue
|
|
# for multi-cluster scheduling. Defaults to the built-in trainjob controller.
|
|
# managedBy: kueue.x-k8s.io/multikueue
|
|
|
|
trainer:
|
|
numNodes: 3
|
|
image: python:3.11-slim
|
|
command:
|
|
- python3
|
|
- -c
|
|
- |
|
|
import time
|
|
print("Training started via Kueue")
|
|
for i in range(10):
|
|
print(f"Step {i + 1}/10")
|
|
time.sleep(2)
|
|
print("Training complete")
|
|
|
|
resourcesPerNode:
|
|
requests:
|
|
cpu: "2"
|
|
memory: "4Gi"
|
|
limits:
|
|
cpu: "4"
|
|
memory: "8Gi"
|
|
|
|
---
|
|
# Reference LocalQueue (apply separately if needed):
|
|
#
|
|
# apiVersion: kueue.x-k8s.io/v1beta1
|
|
# kind: LocalQueue
|
|
# metadata:
|
|
# name: training-queue
|
|
# spec:
|
|
# clusterQueue: cluster-queue
|