mirror of
https://github.com/hanzo-ml/trainer.git
synced 2026-09-24 00:31:34 +00:00
Andrey VelichkevichandGitHub
663ae38567
* feat(runtimes): Add label to disable webhook validation for Runtimes Signed-off-by: Andrey Velichkevich <[email protected]> * Use commonLabels to set labels Signed-off-by: Andrey Velichkevich <[email protected]> * Set resources to base runtimes in E2Es Signed-off-by: Andrey Velichkevich <[email protected]> * Fix Helm runtime images in e2e Signed-off-by: Andrey Velichkevich <[email protected]> --------- Signed-off-by: Andrey Velichkevich <[email protected]>
114 lines
4.0 KiB
YAML
114 lines
4.0 KiB
YAML
{{- /*
|
|
Copyright 2026 The Kubeflow authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/ -}}
|
|
|
|
{{- if or .Values.runtimes.torchtuneDistributed.llama3_2_3B.enabled .Values.runtimes.defaultEnabled }}
|
|
apiVersion: trainer.kubeflow.org/v1alpha1
|
|
kind: ClusterTrainingRuntime
|
|
metadata:
|
|
name: torchtune-llama3.2-3b
|
|
labels:
|
|
trainer.kubeflow.org/framework: torchtune
|
|
{{- with .Values.runtimes.commonLabels }}
|
|
{{- toYaml . | nindent 4 }}
|
|
{{- end }}
|
|
{{- include "trainer.labels" . | nindent 4 }}
|
|
spec:
|
|
mlPolicy:
|
|
numNodes: 1
|
|
torch: {}
|
|
template:
|
|
spec:
|
|
volumeClaimPolicies:
|
|
- templates:
|
|
- metadata:
|
|
name: initializer
|
|
spec:
|
|
accessModes: ["ReadWriteOnce"]
|
|
resources:
|
|
requests:
|
|
storage: 20Gi
|
|
replicatedJobs:
|
|
- name: dataset-initializer
|
|
template:
|
|
metadata:
|
|
labels:
|
|
trainer.kubeflow.org/trainjob-ancestor-step: dataset-initializer
|
|
spec:
|
|
template:
|
|
spec:
|
|
containers:
|
|
- name: dataset-initializer
|
|
image: ghcr.io/kubeflow/trainer/dataset-initializer
|
|
env:
|
|
- name: STORAGE_URI
|
|
value: hf://tatsu-lab/alpaca
|
|
volumeMounts:
|
|
- mountPath: /workspace
|
|
name: initializer
|
|
- name: model-initializer
|
|
template:
|
|
metadata:
|
|
labels:
|
|
trainer.kubeflow.org/trainjob-ancestor-step: model-initializer
|
|
spec:
|
|
template:
|
|
spec:
|
|
containers:
|
|
- name: model-initializer
|
|
image: ghcr.io/kubeflow/trainer/model-initializer
|
|
env:
|
|
- name: STORAGE_URI
|
|
value: hf://meta-llama/Llama-3.2-3B-Instruct
|
|
volumeMounts:
|
|
- name: initializer
|
|
mountPath: /workspace
|
|
- name: node
|
|
dependsOn:
|
|
- name: dataset-initializer
|
|
status: Complete
|
|
- name: model-initializer
|
|
status: Complete
|
|
template:
|
|
metadata:
|
|
labels:
|
|
trainer.kubeflow.org/trainjob-ancestor-step: trainer
|
|
spec:
|
|
template:
|
|
spec:
|
|
containers:
|
|
- name: node
|
|
image: {{ include "trainer.runtimeImage" (list .Values.runtimes.torchtuneDistributed.image .) }}
|
|
command:
|
|
- tune
|
|
- run
|
|
- --rdzv_endpoint=localhost:29500
|
|
- full_finetune_distributed
|
|
- --config
|
|
- llama3_2/3B_full
|
|
- dataset=torchtune.datasets.instruct_dataset
|
|
- dataset.source=parquet
|
|
- dataset.data_dir=/workspace/dataset/data
|
|
- output_dir=/workspace/output
|
|
- tokenizer.path=/workspace/model/original/tokenizer.model
|
|
- checkpointer.checkpoint_dir=/workspace/model
|
|
resources:
|
|
limits:
|
|
nvidia.com/gpu: 2
|
|
volumeMounts:
|
|
- mountPath: /workspace
|
|
name: initializer
|
|
{{- end }}
|