mirror of
https://github.com/hanzo-ml/trainer.git
synced 2026-09-24 00:31:34 +00:00
Tariq HasanandGitHub
1f57df4484
* ci(boilerplate): enforce copyright headers on YAML files Signed-off-by: tariq-hasan <[email protected]> * ci(boilerplate): enforce year-less header for new files Signed-off-by: tariq-hasan <[email protected]> --------- Signed-off-by: tariq-hasan <[email protected]>
60 lines
2.1 KiB
YAML
60 lines
2.1 KiB
YAML
# Copyright The Kubeflow Authors.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# Example 1: Multi-Node Distributed Training
|
|
#
|
|
# Multi-node distributed PyTorch training on the default torch-distributed
|
|
# ClusterTrainingRuntime. The trainer is launched with torchrun, which reads
|
|
# the rendezvous info from the PET_* env vars injected by Trainer
|
|
# (PET_NNODES, PET_NPROC_PER_NODE, PET_NODE_RANK, PET_MASTER_ADDR,
|
|
# PET_MASTER_PORT) and exposes WORLD_SIZE / RANK / LOCAL_RANK to each worker.
|
|
#
|
|
# See https://www.kubeflow.org/docs/components/trainer/user-guides/pytorch/#pytorch-distributed-environment
|
|
#
|
|
# Apply: kubectl apply -f 01-multi-node.yaml
|
|
# Check: kubectl get trainjob multi-node-example
|
|
# Logs: kubectl logs -l jobset.sigs.k8s.io/jobset-name=multi-node-example
|
|
# Clean: kubectl delete trainjob multi-node-example
|
|
|
|
apiVersion: trainer.kubeflow.org/v1alpha1
|
|
kind: TrainJob
|
|
metadata:
|
|
name: multi-node-example
|
|
spec:
|
|
runtimeRef:
|
|
apiGroup: trainer.kubeflow.org
|
|
kind: ClusterTrainingRuntime
|
|
name: torch-distributed
|
|
|
|
trainer:
|
|
numNodes: 3
|
|
command:
|
|
- torchrun
|
|
- --no-python
|
|
- python3
|
|
- -c
|
|
- |
|
|
import os
|
|
import torch
|
|
import torch.distributed as dist
|
|
|
|
device, backend = ("cuda", "nccl") if torch.cuda.is_available() else ("cpu", "gloo")
|
|
dist.init_process_group(backend=backend)
|
|
|
|
print("PyTorch Distributed Environment")
|
|
print(f"Using device: {device}")
|
|
print(f"WORLD_SIZE: {dist.get_world_size()}")
|
|
print(f"RANK: {dist.get_rank()}")
|
|
print(f"LOCAL_RANK: {os.environ['LOCAL_RANK']}")
|