mirror of
https://github.com/hanzo-ml/trainer.git
synced 2026-09-24 00:31:34 +00:00
Tariq HasanandGitHub
1f57df4484
* ci(boilerplate): enforce copyright headers on YAML files Signed-off-by: tariq-hasan <[email protected]> * ci(boilerplate): enforce year-less header for new files Signed-off-by: tariq-hasan <[email protected]> --------- Signed-off-by: tariq-hasan <[email protected]>
97 lines
2.9 KiB
YAML
97 lines
2.9 KiB
YAML
# Copyright The Kubeflow Authors.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# Advanced Example 3: Volcano Gang Scheduling
|
|
#
|
|
# Gang scheduling guarantees that all training pods start together (or none
|
|
# do), avoiding the deadlocks that can occur when a distributed job is
|
|
# partially admitted.
|
|
#
|
|
# podGroupPolicy is a field on the TrainingRuntime, not the TrainJob, so this
|
|
# example creates a Volcano-enabled ClusterTrainingRuntime and a TrainJob
|
|
# that references it.
|
|
#
|
|
# Prerequisites:
|
|
# - Volcano scheduler installed in the cluster
|
|
#
|
|
# Apply: kubectl apply -f 04-volcano-integration.yaml
|
|
# Check: kubectl get trainjob volcano-example
|
|
# Pods: kubectl get pods -l jobset.sigs.k8s.io/jobset-name=volcano-example
|
|
# Group: kubectl get podgroup -l jobset.sigs.k8s.io/jobset-name=volcano-example
|
|
# Clean: kubectl delete trainjob volcano-example && \
|
|
# kubectl delete clustertrainingruntime torch-distributed-volcano
|
|
#
|
|
# Reference: https://www.kubeflow.org/docs/components/trainer/operator-guides/job-scheduling/volcano/
|
|
|
|
apiVersion: trainer.kubeflow.org/v1alpha1
|
|
kind: ClusterTrainingRuntime
|
|
metadata:
|
|
name: torch-distributed-volcano
|
|
labels:
|
|
trainer.kubeflow.org/framework: torch
|
|
spec:
|
|
mlPolicy:
|
|
numNodes: 1
|
|
torch: {}
|
|
podGroupPolicy:
|
|
volcano: {}
|
|
template:
|
|
spec:
|
|
replicatedJobs:
|
|
- name: node
|
|
template:
|
|
metadata:
|
|
labels:
|
|
trainer.kubeflow.org/trainjob-ancestor-step: trainer
|
|
spec:
|
|
template:
|
|
spec:
|
|
containers:
|
|
- name: node
|
|
image: pytorch/pytorch:2.10.0-cuda12.8-cudnn9-runtime
|
|
|
|
---
|
|
apiVersion: trainer.kubeflow.org/v1alpha1
|
|
kind: TrainJob
|
|
metadata:
|
|
name: volcano-example
|
|
spec:
|
|
runtimeRef:
|
|
apiGroup: trainer.kubeflow.org
|
|
kind: ClusterTrainingRuntime
|
|
name: torch-distributed-volcano
|
|
|
|
trainer:
|
|
numNodes: 4
|
|
command:
|
|
- python3
|
|
- -c
|
|
- |
|
|
import os
|
|
import time
|
|
node_rank = os.getenv("PET_NODE_RANK", "0")
|
|
nnodes = os.getenv("PET_NNODES", "1")
|
|
print(f"Node {node_rank}/{nnodes} started via Volcano gang scheduling")
|
|
print(f"All {nnodes} nodes are guaranteed to start together")
|
|
time.sleep(5)
|
|
print(f"Node {node_rank} training complete")
|
|
|
|
resourcesPerNode:
|
|
requests:
|
|
cpu: "1"
|
|
memory: "2Gi"
|
|
limits:
|
|
cpu: "2"
|
|
memory: "4Gi"
|