mirror of
https://github.com/hanzo-ml/trainer.git
synced 2026-09-24 00:31:34 +00:00
* feat(api): add progress types to api Signed-off-by: Rob Bell <[email protected]> * feat: progress server stub Signed-off-by: Rob Bell <[email protected]> * feat: setup tls for server Signed-off-by: Rob Bell <[email protected]> * refactor: pull out middleware Signed-off-by: Rob Bell <[email protected]> * refactor: remove logger message noise from tests Signed-off-by: Rob Bell <[email protected]> * fix: send server error logs to main logger Signed-off-by: Rob Bell <[email protected]> * feat: update the trainer status Signed-off-by: Rob Bell <[email protected]> * feat: use separate client to avoid impacting main reconciler client-side rate limits Signed-off-by: Rob Bell <[email protected]> * feat: configure progress server in helm chart and configmap Signed-off-by: Rob Bell <[email protected]> * fix: prevent race condition Signed-off-by: Rob Bell <[email protected]> * feat: inject progress runtime config Signed-off-by: Rob Bell <[email protected]> * fix: progress client config Signed-off-by: Rob Bell <[email protected]> * fix: update generated config Signed-off-by: Rob Bell <[email protected]> * feat: validate auth tokens in endpoint Signed-off-by: Rob Bell <[email protected]> * test: validate auth tokens in endpoint Signed-off-by: Rob Bell <[email protected]> * refactor: tidy tests to use same format as other tests Signed-off-by: Rob Bell <[email protected]> * fix: look up the cluster issuer from the service account token Different clusters can use different issuers so look up the issuer from the trusted in-cluster service account token. Signed-off-by: Rob Bell <[email protected]> * refactor: rename authn middleware Signed-off-by: Rob Bell <[email protected]> * feat: add authn to status update endpoint Signed-off-by: Rob Bell <[email protected]> * refactor: tidy up reading in-cluster service account token Signed-off-by: Rob Bell <[email protected]> * fix: rbac permissions Signed-off-by: Rob Bell <[email protected]> * fix: better error response Signed-off-by: Rob Bell <[email protected]> * refactor: move kubebuilder annotation higher up Signed-off-by: Rob Bell <[email protected]> * fix: make lastUpdateTime required Signed-off-by: Rob Bell <[email protected]> * chore: fix config tests after rebase Signed-off-by: Rob Bell <[email protected]> * test: update config test with new progress server config Signed-off-by: Rob Bell <[email protected]> * test: add progress tracking e2e test Signed-off-by: Rob Bell <[email protected]> * fix: allow feature gates to be configurable from command line argument So they can be overridden in e2e tests Signed-off-by: Rob Bell <[email protected]> * refactor: reuse shared progress StatusUrl Signed-off-by: Rob Bell <[email protected]> * test: tidy up tests Signed-off-by: Rob Bell <[email protected]> * refactor: allow plugins to use config Signed-off-by: Rob Bell <[email protected]> * refactor: centralise shared constants Signed-off-by: Rob Bell <[email protected]> * fix: disable http2 in progress server tls opts if disabled by config Signed-off-by: Rob Bell <[email protected]> * fix: minor improvements suggested by copilot Signed-off-by: Rob Bell <[email protected]> * chore: rename progress server => status server Signed-off-by: Rob Bell <[email protected]> * chore: update type doc Signed-off-by: Rob Bell <[email protected]> * chore: update name of feature gate Signed-off-by: Rob Bell <[email protected]> * fix: after rebase Signed-off-by: Rob Bell <[email protected]> * chore: fix typo Signed-off-by: Rob Bell <[email protected]> * chore: rename handler Signed-off-by: Rob Bell <[email protected]> * fix: run status server on all replicas Signed-off-by: Rob Bell <[email protected]> * fix: deep copy the rest config Signed-off-by: Rob Bell <[email protected]> * refactor: use ssa Signed-off-by: Rob Bell <[email protected]> * refactor: separate token audience per train job Signed-off-by: Rob Bell <[email protected]> * chore: rename status server Signed-off-by: Rob Bell <[email protected]> * fix: revert adding status pkg to manifest gen paths Signed-off-by: Rob Bell <[email protected]> * chore: update log messages Signed-off-by: Rob Bell <[email protected]> * chore: update api docs Signed-off-by: Rob Bell <[email protected]> * chore: update format Signed-off-by: Rob Bell <[email protected]> * fix: revert inadvertent change Signed-off-by: Rob Bell <[email protected]> * chore: update api name and type docs Signed-off-by: Rob Bell <[email protected]> * refactor: initialise oidc provider in manager startup Signed-off-by: Rob Bell <[email protected]> * fix: allow empty update request Signed-off-by: Rob Bell <[email protected]> * fix: wait for server shut down Signed-off-by: Rob Bell <[email protected]> * fix: handle missing train job Signed-off-by: Rob Bell <[email protected]> * refactor: update env var names Signed-off-by: Rob Bell <[email protected]> * chore: rename packages, plugins, feature gate Signed-off-by: Rob Bell <[email protected]> * refactor: update env var names Signed-off-by: Rob Bell <[email protected]> * chore: regenerate after rebase Signed-off-by: Rob Bell <[email protected]> * chore: fixes after rebase Signed-off-by: Rob Bell <[email protected]> * chore: minor doc updates Signed-off-by: Rob Bell <[email protected]> * chore: rename status server config Signed-off-by: Rob Bell <[email protected]> * chore: replace placeholder config with nil Signed-off-by: Rob Bell <[email protected]> * chore: revert separate runtime utils package Signed-off-by: Rob Bell <[email protected]> * chore: remove unused config from tests Signed-off-by: Rob Bell <[email protected]> --------- Signed-off-by: Rob Bell <[email protected]>
60 lines
2.1 KiB
Go
60 lines
2.1 KiB
Go
/*
|
|
Copyright 2024 The Kubeflow Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package plugins
|
|
|
|
import (
|
|
"context"
|
|
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
|
|
configapi "github.com/kubeflow/trainer/v2/pkg/apis/config/v1alpha1"
|
|
"github.com/kubeflow/trainer/v2/pkg/features"
|
|
"github.com/kubeflow/trainer/v2/pkg/runtime/framework"
|
|
"github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/coscheduling"
|
|
"github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/flux"
|
|
"github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/jax"
|
|
"github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/jobset"
|
|
"github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/mpi"
|
|
"github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/plainml"
|
|
"github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/torch"
|
|
"github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/trainjobstatus"
|
|
"github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/volcano"
|
|
"github.com/kubeflow/trainer/v2/pkg/runtime/framework/plugins/xgboost"
|
|
)
|
|
|
|
type Registry map[string]func(ctx context.Context, client client.Client, indexer client.FieldIndexer, cfg *configapi.Configuration) (framework.Plugin, error)
|
|
|
|
func NewRegistry() Registry {
|
|
registry := Registry{
|
|
coscheduling.Name: coscheduling.New,
|
|
flux.Name: flux.New,
|
|
volcano.Name: volcano.New,
|
|
mpi.Name: mpi.New,
|
|
plainml.Name: plainml.New,
|
|
torch.Name: torch.New,
|
|
jobset.Name: jobset.New,
|
|
jax.Name: jax.New,
|
|
xgboost.Name: xgboost.New,
|
|
}
|
|
|
|
if features.Enabled(features.TrainJobStatus) {
|
|
registry[trainjobstatus.Name] = trainjobstatus.New
|
|
}
|
|
|
|
return registry
|
|
}
|