mirror of
https://github.com/hanzo-ml/trainer.git
synced 2026-09-24 00:31:34 +00:00
88a92a0483
* feat: support for flux framework as hpc manager Flux supports the majority of MPI flavors/variants, and can be used to bootstrap MPI as a plugin. It adds other features for scheduling and topology that can be used for simulations and ai/ml jobs. This changeset adds the plugin implementation, including the plugin module, tests, and an example with a small README to serve as documentation for the time being. Signed-off-by: vsoch <[email protected]> * review: slim down use of annotations We still need to put the original command in an annotation to retieve later, but others can be re-derived from the environment Signed-off-by: vsoch <[email protected]> * review: clean up tasks and remove extra FluxPolicySource Signed-off-by: vsoch <[email protected]> * review: ensure to unset trainer args Signed-off-by: vsoch <[email protected]> * review: better support for nproc per node and nodes Signed-off-by: vsoch <[email protected]> * review: add constants types for flux Signed-off-by: vsoch <[email protected]> * review: add gpu specification from resources Signed-off-by: vsoch <[email protected]> * Fix TrainJob NumProcPerNode Signed-off-by: Andrey Velichkevich <[email protected]> * Remove elasticPolicy from kubebuilder:validation Signed-off-by: Andrey Velichkevich <[email protected]> * chore: use go embed for long string templates flux plugin Signed-off-by: vsoch <[email protected]> * cleanup: removed un-used annotation Signed-off-by: vsoch <[email protected]> --------- Signed-off-by: vsoch <[email protected]> Signed-off-by: Andrey Velichkevich <[email protected]> Co-authored-by: vsoch <[email protected]> Co-authored-by: Andrey Velichkevich <[email protected]>
415 lines
15 KiB
Go
415 lines
15 KiB
Go
/*
|
|
Copyright 2025 The Kubeflow Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package webhooks
|
|
|
|
import (
|
|
"github.com/onsi/ginkgo/v2"
|
|
"github.com/onsi/gomega"
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/utils/ptr"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
|
|
trainer "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1"
|
|
"github.com/kubeflow/trainer/v2/pkg/constants"
|
|
testingutil "github.com/kubeflow/trainer/v2/pkg/util/testing"
|
|
"github.com/kubeflow/trainer/v2/test/integration/framework"
|
|
"github.com/kubeflow/trainer/v2/test/util"
|
|
)
|
|
|
|
const trainingRuntimeName = "test-trainingruntime"
|
|
|
|
var _ = ginkgo.Describe("TrainingRuntime Webhook", ginkgo.Ordered, func() {
|
|
var ns *corev1.Namespace
|
|
|
|
ginkgo.BeforeAll(func() {
|
|
fwk = &framework.Framework{}
|
|
cfg = fwk.Init()
|
|
ctx, k8sClient = fwk.RunManager(cfg, false)
|
|
})
|
|
ginkgo.AfterAll(func() {
|
|
fwk.Teardown()
|
|
})
|
|
|
|
ginkgo.BeforeEach(func() {
|
|
ns = &corev1.Namespace{
|
|
TypeMeta: metav1.TypeMeta{
|
|
APIVersion: corev1.SchemeGroupVersion.String(),
|
|
Kind: "Namespace",
|
|
},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
GenerateName: "trainingruntime-webhook-",
|
|
},
|
|
}
|
|
gomega.Expect(k8sClient.Create(ctx, ns)).To(gomega.Succeed())
|
|
})
|
|
|
|
ginkgo.AfterEach(func() {
|
|
gomega.Expect(k8sClient.DeleteAllOf(ctx, &trainer.TrainingRuntime{}, client.InNamespace(ns.Name))).To(gomega.Succeed())
|
|
})
|
|
|
|
ginkgo.When("Creating TrainingRuntime", func() {
|
|
ginkgo.DescribeTable("Validate TrainingRuntime on creation", func(runtime func() *trainer.TrainingRuntime) {
|
|
gomega.Expect(k8sClient.Create(ctx, runtime())).Should(gomega.Succeed())
|
|
},
|
|
ginkgo.Entry("Should succeed to create TrainingRuntime",
|
|
func() *trainer.TrainingRuntime {
|
|
baseRuntime := testingutil.MakeTrainingRuntimeWrapper(ns.Name, trainingRuntimeName)
|
|
return baseRuntime.
|
|
RuntimeSpec(
|
|
testingutil.MakeTrainingRuntimeSpecWrapper(baseRuntime.Spec).
|
|
Obj()).
|
|
Obj()
|
|
}),
|
|
)
|
|
})
|
|
})
|
|
|
|
var _ = ginkgo.Describe("TrainingRuntime marker validations and defaulting", ginkgo.Ordered, func() {
|
|
var ns *corev1.Namespace
|
|
|
|
ginkgo.BeforeAll(func() {
|
|
fwk = &framework.Framework{}
|
|
cfg = fwk.Init()
|
|
ctx, k8sClient = fwk.RunManager(cfg, false)
|
|
})
|
|
ginkgo.AfterAll(func() {
|
|
fwk.Teardown()
|
|
})
|
|
|
|
ginkgo.BeforeEach(func() {
|
|
ns = &corev1.Namespace{
|
|
TypeMeta: metav1.TypeMeta{
|
|
APIVersion: corev1.SchemeGroupVersion.String(),
|
|
Kind: "Namespace",
|
|
},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
GenerateName: "training-runtime-marker-",
|
|
},
|
|
}
|
|
gomega.Expect(k8sClient.Create(ctx, ns)).To(gomega.Succeed())
|
|
})
|
|
ginkgo.AfterEach(func() {
|
|
gomega.Expect(k8sClient.DeleteAllOf(ctx, &trainer.TrainingRuntime{}, client.InNamespace(ns.Name))).Should(gomega.Succeed())
|
|
gomega.Expect(k8sClient.DeleteAllOf(ctx, &trainer.ClusterTrainingRuntime{})).Should(gomega.Succeed())
|
|
})
|
|
|
|
ginkgo.When("Creating TrainingRuntime", func() {
|
|
ginkgo.DescribeTable("Validate TrainingRuntime on creation", func(trainingRuntime func() *trainer.TrainingRuntime, errorMatcher gomega.OmegaMatcher) {
|
|
gomega.Expect(k8sClient.Create(ctx, trainingRuntime())).Should(errorMatcher)
|
|
},
|
|
ginkgo.Entry("Should succeed to create trainingRuntime",
|
|
func() *trainer.TrainingRuntime {
|
|
return testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").
|
|
Obj()
|
|
},
|
|
gomega.Succeed()),
|
|
ginkgo.Entry("Should fail to create trainingRuntime with both MPI and Torch runtimes",
|
|
func() *trainer.TrainingRuntime {
|
|
runtime := testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").Obj()
|
|
runtime.Spec.MLPolicy = &trainer.MLPolicy{
|
|
MLPolicySource: trainer.MLPolicySource{
|
|
Torch: &trainer.TorchMLPolicySource{},
|
|
MPI: &trainer.MPIMLPolicySource{},
|
|
},
|
|
}
|
|
return runtime
|
|
},
|
|
testingutil.BeInvalidError()),
|
|
ginkgo.Entry("Should fail to create trainingRuntime with both JAX and Torch runtimes",
|
|
func() *trainer.TrainingRuntime {
|
|
runtime := testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").Obj()
|
|
runtime.Spec.MLPolicy = &trainer.MLPolicy{
|
|
MLPolicySource: trainer.MLPolicySource{
|
|
Torch: &trainer.TorchMLPolicySource{},
|
|
JAX: &trainer.JAXMLPolicySource{},
|
|
},
|
|
}
|
|
return runtime
|
|
},
|
|
testingutil.BeInvalidError()),
|
|
ginkgo.Entry("Should fail to create trainingRuntime with both Torch and Flux runtimes",
|
|
func() *trainer.TrainingRuntime {
|
|
runtime := testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").Obj()
|
|
runtime.Spec.MLPolicy = &trainer.MLPolicy{
|
|
MLPolicySource: trainer.MLPolicySource{
|
|
Torch: &trainer.TorchMLPolicySource{},
|
|
Flux: &trainer.FluxMLPolicySource{},
|
|
},
|
|
}
|
|
return runtime
|
|
},
|
|
testingutil.BeInvalidError()),
|
|
ginkgo.Entry("Should fail to create trainingRuntime with both MPI and Flux runtimes",
|
|
func() *trainer.TrainingRuntime {
|
|
runtime := testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").Obj()
|
|
runtime.Spec.MLPolicy = &trainer.MLPolicy{
|
|
MLPolicySource: trainer.MLPolicySource{
|
|
MPI: &trainer.MPIMLPolicySource{},
|
|
Flux: &trainer.FluxMLPolicySource{},
|
|
},
|
|
}
|
|
return runtime
|
|
},
|
|
testingutil.BeInvalidError()),
|
|
ginkgo.Entry("Should fail to create trainingRuntime with both JAX and Flux runtimes",
|
|
func() *trainer.TrainingRuntime {
|
|
runtime := testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").Obj()
|
|
runtime.Spec.MLPolicy = &trainer.MLPolicy{
|
|
MLPolicySource: trainer.MLPolicySource{
|
|
Flux: &trainer.FluxMLPolicySource{},
|
|
JAX: &trainer.JAXMLPolicySource{},
|
|
},
|
|
}
|
|
return runtime
|
|
},
|
|
testingutil.BeInvalidError()),
|
|
ginkgo.Entry("Should fail to create trainingRuntime with both JAX and MPI runtimes",
|
|
func() *trainer.TrainingRuntime {
|
|
runtime := testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").Obj()
|
|
runtime.Spec.MLPolicy = &trainer.MLPolicy{
|
|
MLPolicySource: trainer.MLPolicySource{
|
|
MPI: &trainer.MPIMLPolicySource{},
|
|
JAX: &trainer.JAXMLPolicySource{},
|
|
},
|
|
}
|
|
return runtime
|
|
},
|
|
testingutil.BeInvalidError()),
|
|
ginkgo.Entry("Should fail to create trainingRuntime with all four runtimes configured",
|
|
func() *trainer.TrainingRuntime {
|
|
runtime := testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").Obj()
|
|
runtime.Spec.MLPolicy = &trainer.MLPolicy{
|
|
MLPolicySource: trainer.MLPolicySource{
|
|
Torch: &trainer.TorchMLPolicySource{},
|
|
MPI: &trainer.MPIMLPolicySource{},
|
|
JAX: &trainer.JAXMLPolicySource{},
|
|
Flux: &trainer.FluxMLPolicySource{},
|
|
},
|
|
}
|
|
return runtime
|
|
},
|
|
testingutil.BeInvalidError()),
|
|
ginkgo.Entry("Should fail to create trainingRuntime with non-supported mpi.mpiImplementation",
|
|
func() *trainer.TrainingRuntime {
|
|
return testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").
|
|
RuntimeSpec(testingutil.MakeTrainingRuntimeSpecWrapper(
|
|
testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").Obj().Spec).
|
|
WithMLPolicy(
|
|
testingutil.MakeMLPolicyWrapper().
|
|
WithMLPolicySource(*testingutil.MakeMLPolicySourceWrapper().
|
|
MPIPolicy(nil, "invalid", nil, nil).
|
|
Obj(),
|
|
).
|
|
Obj(),
|
|
).
|
|
Obj()).
|
|
Obj()
|
|
},
|
|
testingutil.BeInvalidError(),
|
|
),
|
|
)
|
|
ginkgo.DescribeTable("Defaulting TrainingRuntime on creation", func(trainingRuntime func() *trainer.TrainingRuntime, wantTrainingRuntime func() *trainer.TrainingRuntime) {
|
|
created := trainingRuntime()
|
|
gomega.Expect(k8sClient.Create(ctx, created)).Should(gomega.Succeed())
|
|
gomega.Expect(created).Should(gomega.BeComparableTo(wantTrainingRuntime(), util.IgnoreObjectMetadata))
|
|
},
|
|
|
|
ginkgo.Entry("Should succeed to default mpi.mpiImplementation=OpenMPI",
|
|
func() *trainer.TrainingRuntime {
|
|
return testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").
|
|
RuntimeSpec(testingutil.MakeTrainingRuntimeSpecWrapper(
|
|
testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").Obj().Spec).
|
|
WithMLPolicy(
|
|
testingutil.MakeMLPolicyWrapper().
|
|
WithMLPolicySource(*testingutil.MakeMLPolicySourceWrapper().
|
|
MPIPolicy(ptr.To[int32](1), trainer.MPIImplementationOpenMPI, ptr.To("/usr/dir"), ptr.To(false)).
|
|
Obj(),
|
|
).
|
|
Obj(),
|
|
).
|
|
JobSetSpec(
|
|
testingutil.MakeJobSetWrapper(ns.Name, "jobset").
|
|
Replicas(1, constants.Node, constants.DatasetInitializer, constants.ModelInitializer, constants.Launcher).
|
|
Obj().
|
|
Spec,
|
|
).
|
|
Obj(),
|
|
).
|
|
Obj()
|
|
},
|
|
func() *trainer.TrainingRuntime {
|
|
return testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").
|
|
RuntimeSpec(testingutil.MakeTrainingRuntimeSpecWrapper(
|
|
testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").Obj().Spec).
|
|
WithMLPolicy(
|
|
testingutil.MakeMLPolicyWrapper().
|
|
WithMLPolicySource(*testingutil.MakeMLPolicySourceWrapper().
|
|
MPIPolicy(ptr.To[int32](1), trainer.MPIImplementationOpenMPI, ptr.To("/usr/dir"), ptr.To(false)).
|
|
Obj(),
|
|
).
|
|
Obj(),
|
|
).
|
|
JobSetSpec(
|
|
testingutil.MakeJobSetWrapper(ns.Name, "jobset").
|
|
Replicas(1, constants.Node, constants.DatasetInitializer, constants.ModelInitializer, constants.Launcher).
|
|
Obj().
|
|
Spec,
|
|
).
|
|
Obj(),
|
|
).
|
|
Obj()
|
|
}),
|
|
ginkgo.Entry("Should succeed to default mpi.sshAuthMountPath=/root/.ssh",
|
|
func() *trainer.TrainingRuntime {
|
|
return testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").
|
|
RuntimeSpec(testingutil.MakeTrainingRuntimeSpecWrapper(
|
|
testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").Obj().Spec).
|
|
WithMLPolicy(
|
|
testingutil.MakeMLPolicyWrapper().
|
|
WithMLPolicySource(*testingutil.MakeMLPolicySourceWrapper().
|
|
MPIPolicy(ptr.To[int32](1), trainer.MPIImplementationOpenMPI, nil, ptr.To(false)).
|
|
Obj(),
|
|
).
|
|
Obj(),
|
|
).
|
|
JobSetSpec(
|
|
testingutil.MakeJobSetWrapper(ns.Name, "jobset").
|
|
Replicas(1, constants.Node, constants.DatasetInitializer, constants.ModelInitializer, constants.Launcher).
|
|
Obj().
|
|
Spec,
|
|
).
|
|
Obj(),
|
|
).
|
|
Obj()
|
|
},
|
|
func() *trainer.TrainingRuntime {
|
|
return testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").
|
|
RuntimeSpec(testingutil.MakeTrainingRuntimeSpecWrapper(
|
|
testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").Obj().Spec).
|
|
WithMLPolicy(
|
|
testingutil.MakeMLPolicyWrapper().
|
|
WithMLPolicySource(*testingutil.MakeMLPolicySourceWrapper().
|
|
MPIPolicy(ptr.To[int32](1), trainer.MPIImplementationOpenMPI, ptr.To("/root/.ssh"), ptr.To(false)).
|
|
Obj(),
|
|
).
|
|
Obj(),
|
|
).
|
|
JobSetSpec(
|
|
testingutil.MakeJobSetWrapper(ns.Name, "jobset").
|
|
Replicas(1, constants.Node, constants.DatasetInitializer, constants.ModelInitializer, constants.Launcher).
|
|
Obj().
|
|
Spec,
|
|
).
|
|
Obj(),
|
|
).
|
|
Obj()
|
|
}),
|
|
ginkgo.Entry("Should succeed to default mpi.runLauncherAsNode=false",
|
|
func() *trainer.TrainingRuntime {
|
|
return testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").
|
|
RuntimeSpec(testingutil.MakeTrainingRuntimeSpecWrapper(
|
|
testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").Obj().Spec).
|
|
WithMLPolicy(
|
|
testingutil.MakeMLPolicyWrapper().
|
|
WithMLPolicySource(*testingutil.MakeMLPolicySourceWrapper().
|
|
MPIPolicy(ptr.To[int32](1), trainer.MPIImplementationOpenMPI, ptr.To("/usr/dir"), nil).
|
|
Obj(),
|
|
).
|
|
Obj(),
|
|
).
|
|
JobSetSpec(
|
|
testingutil.MakeJobSetWrapper(ns.Name, "jobset").
|
|
Replicas(1, constants.Node, constants.DatasetInitializer, constants.ModelInitializer, constants.Launcher).
|
|
Obj().
|
|
Spec,
|
|
).
|
|
Obj(),
|
|
).
|
|
Obj()
|
|
},
|
|
func() *trainer.TrainingRuntime {
|
|
return testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").
|
|
RuntimeSpec(testingutil.MakeTrainingRuntimeSpecWrapper(
|
|
testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").Obj().Spec).
|
|
WithMLPolicy(
|
|
testingutil.MakeMLPolicyWrapper().
|
|
WithMLPolicySource(*testingutil.MakeMLPolicySourceWrapper().
|
|
MPIPolicy(ptr.To[int32](1), trainer.MPIImplementationOpenMPI, ptr.To("/usr/dir"), ptr.To(false)).
|
|
Obj(),
|
|
).
|
|
Obj(),
|
|
).
|
|
JobSetSpec(
|
|
testingutil.MakeJobSetWrapper(ns.Name, "jobset").
|
|
Replicas(1, constants.Node, constants.DatasetInitializer, constants.ModelInitializer, constants.Launcher).
|
|
Obj().
|
|
Spec,
|
|
).
|
|
Obj(),
|
|
).
|
|
Obj()
|
|
}),
|
|
ginkgo.Entry("Should succeed to default mpi.numProcPerNode=1",
|
|
func() *trainer.TrainingRuntime {
|
|
return testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").
|
|
RuntimeSpec(testingutil.MakeTrainingRuntimeSpecWrapper(
|
|
testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").Obj().Spec).
|
|
WithMLPolicy(
|
|
testingutil.MakeMLPolicyWrapper().
|
|
WithMLPolicySource(*testingutil.MakeMLPolicySourceWrapper().
|
|
MPIPolicy(nil, trainer.MPIImplementationOpenMPI, ptr.To("/usr/dir"), ptr.To(false)).
|
|
Obj(),
|
|
).
|
|
Obj(),
|
|
).
|
|
JobSetSpec(
|
|
testingutil.MakeJobSetWrapper(ns.Name, "jobset").
|
|
Replicas(1, constants.Node, constants.DatasetInitializer, constants.ModelInitializer, constants.Launcher).
|
|
Obj().
|
|
Spec,
|
|
).
|
|
Obj(),
|
|
).
|
|
Obj()
|
|
},
|
|
func() *trainer.TrainingRuntime {
|
|
return testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").
|
|
RuntimeSpec(testingutil.MakeTrainingRuntimeSpecWrapper(
|
|
testingutil.MakeTrainingRuntimeWrapper(ns.Name, "runtime").Obj().Spec).
|
|
WithMLPolicy(
|
|
testingutil.MakeMLPolicyWrapper().
|
|
WithMLPolicySource(*testingutil.MakeMLPolicySourceWrapper().
|
|
MPIPolicy(ptr.To[int32](1), trainer.MPIImplementationOpenMPI, ptr.To("/usr/dir"), ptr.To(false)).
|
|
Obj(),
|
|
).
|
|
Obj(),
|
|
).
|
|
JobSetSpec(
|
|
testingutil.MakeJobSetWrapper(ns.Name, "jobset").
|
|
Replicas(1, constants.Node, constants.DatasetInitializer, constants.ModelInitializer, constants.Launcher).
|
|
Obj().
|
|
Spec,
|
|
).
|
|
Obj(),
|
|
).
|
|
Obj()
|
|
}),
|
|
)
|
|
})
|
|
})
|