hanzo-ml/trainer
1
0
Fork 0
mirror of https://github.com/hanzo-ml/trainer.git synced 2026-09-24 00:31:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(api): Replace PodTemplateOverrides with RuntimePatches API (#3309)

Browse Source 
* feat(api): Replace PodTemplateOverrides with RuntimePatches API

Signed-off-by: Andrey Velichkevich <[email protected]>

* Update goal with RuntimePatches API

Signed-off-by: Andrey Velichkevich <[email protected]>

* Add diagram for RuntimePatches

Signed-off-by: Andrey Velichkevich <[email protected]>

* feat(api): Replace PodTemplateOverrides with RuntimePatches API

Signed-off-by: Antonin Stefanutti <[email protected]>

* Fix kube lint errors

Signed-off-by: Antonin Stefanutti <[email protected]>

* Update Python models

Signed-off-by: Antonin Stefanutti <[email protected]>

* Enable maxlength Kube-API-Linter rule

Signed-off-by: Antonin Stefanutti <[email protected]>

* Restore manger field IsDomainPrefixedPath validation

Signed-off-by: Antonin Stefanutti <[email protected]>

* Fix integration tests

Signed-off-by: Antonin Stefanutti <[email protected]>

* Apply StrategicMergePatch to batchv1.JobTemplateSpec

Signed-off-by: Antonin Stefanutti <[email protected]>

---------

Signed-off-by: Andrey Velichkevich <[email protected]>
Signed-off-by: Antonin Stefanutti <[email protected]>
Co-authored-by: Andrey Velichkevich <[email protected]>
This commit is contained in:
Antonin Stefanutti
2026-03-12 16:29:44 +00:00
committed by GitHub
co-authored by Andrey Velichkevich
parent f105c84c58
commit 37c8a07a8a
52 changed files with 14122 additions and 9852 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.golangci-kal.yml
+1 -1
View File
@@ -20,7 +20,7 @@ linters:
#- "forbiddenmarkers" # Ensure that types and fields do not contain any markers that are forbidden.
- "integers" # Ensure only int32 and int64 are used for integers.
- "jsontags" # Ensure every field has a json tag.
#- "maxlength" # Ensure all strings and arrays have maximum lengths/maximum items.
- "maxlength" # Ensure all strings and arrays have maximum lengths/maximum items.
#- "nobools" # Bools do not evolve over time, should use enums instead.
- "nodurations" # Prevents usage of `Duration` types.
#- "nofloats" # Ensure floats are not used.
api/openapi-spec/swagger.json
Generated
+192 -93
View File
@@ -13721,15 +13721,15 @@
}
}
},
"trainer.v1alpha1.ContainerOverride": {
"description": "ContainerOverride represents parameters that can be overridden using PodSpecOverrides.",
"trainer.v1alpha1.ContainerPatch": {
"description": "ContainerPatch represents parameters that can be patched using PodSpecPatch.",
"type": "object",
"required": [
"name"
],
"properties": {
"env": {
"description": "env is the list of environment variables to set in the container. These values will be merged with the TrainingRuntime's environments. These values can't be set for container with the name: `node`, `dataset-initializer`, or `model-initializer`. For those containers the envs can only be set via Trainer or Initializer APIs.",
"description": "env is the list of environment variables to set in the container. These values will be merged with the Runtime's environments. These values can't be set for container with the name: `node`, `dataset-initializer`, or `model-initializer`. For those containers the envs can only be set via Trainer or Initializer APIs.",
"type": "array",
"items": {
"default": {},
@@ -13745,11 +13745,11 @@
"x-kubernetes-list-type": "map"
},
"name": {
"description": "name for the container. TrainingRuntime must have this container.",
"description": "name for the container. Runtime must have this container.",
"type": "string"
},
"securityContext": {
"description": "securityContext overrides the container's security context. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/",
"description": "securityContext patches the container's security context. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/",
"allOf": [
{
"$ref": "#/components/schemas/io.k8s.api.core.v1.SecurityContext"
@@ -13856,6 +13856,50 @@
"description": "JAXMLPolicySource represents a jax runtime configuration.",
"type": "object"
},
"trainer.v1alpha1.JobSetSpecPatch": {
"description": "JobSetSpecPatch defines allowed patches for the JobSet spec.",
"type": "object",
"properties": {
"replicatedJobs": {
"description": "replicatedJobs defines per-job patches, keyed by job name.",
"type": "array",
"items": {
"default": {},
"allOf": [
{
"$ref": "#/components/schemas/trainer.v1alpha1.ReplicatedJobPatch"
}
]
},
"x-kubernetes-list-map-keys": [
"name"
],
"x-kubernetes-list-type": "map"
}
}
},
"trainer.v1alpha1.JobSetTemplatePatch": {
"description": "JobSetTemplatePatch defines patches for the JobSet template. It mirrors JobSetTemplateSpec but only exposes metadata and restricted spec fields.",
"type": "object",
"properties": {
"metadata": {
"description": "metadata patches the JobSet object metadata. Only labels and annotations are allowed.",
"allOf": [
{
"$ref": "#/components/schemas/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"
}
]
},
"spec": {
"description": "spec patches the JobSet spec with restricted fields.",
"allOf": [
{
"$ref": "#/components/schemas/trainer.v1alpha1.JobSetSpecPatch"
}
]
}
}
},
"trainer.v1alpha1.JobSetTemplateSpec": {
"description": "JobSetTemplateSpec represents a template of the desired JobSet.",
"type": "object",
@@ -13880,6 +13924,20 @@
}
}
},
"trainer.v1alpha1.JobSpecPatch": {
"description": "JobSpecPatch defines allowed patches for the Job spec.",
"type": "object",
"properties": {
"template": {
"description": "template patches the Pod template for this Job.",
"allOf": [
{
"$ref": "#/components/schemas/trainer.v1alpha1.PodTemplatePatch"
}
]
}
}
},
"trainer.v1alpha1.JobStatus": {
"type": "object",
"required": [
@@ -13922,6 +13980,28 @@
}
}
},
"trainer.v1alpha1.JobTemplatePatch": {
"description": "JobTemplatePatch defines patches for a Job template within a replicated job.",
"type": "object",
"properties": {
"metadata": {
"description": "metadata patches the Job template metadata. Only labels and annotations are allowed.",
"allOf": [
{
"$ref": "#/components/schemas/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"
}
]
},
"spec": {
"description": "spec patches the Job spec with restricted fields.",
"allOf": [
{
"$ref": "#/components/schemas/trainer.v1alpha1.JobSpecPatch"
}
]
}
}
},
"trainer.v1alpha1.MLPolicy": {
"description": "MLPolicy represents configuration for the model training with ML-specific parameters.",
"type": "object",
@@ -14120,66 +14200,12 @@
}
}
},
"trainer.v1alpha1.PodTemplateOverride": {
"description": "PodTemplateOverride represents a custom PodTemplateSpec override that will be applied to the TrainJob's training runtime.",
"type": "object",
"required": [
"targetJobs"
],
"properties": {
"manager": {
"description": "manager indicates which controller created this PodTemplateOverride object. It can be used by external controllers or admission webhooks to track ownership and avoid conflicts with other controllers. For example, Kueue sets this field to \"kueue.k8s.io/manager\". Defaults to `trainer.kubeflow.org/unknown`",
"type": "string"
},
"metadata": {
"description": "metadata overrides the Pod template metadata. These values will be merged with the TrainingRuntime's Pod template metadata.",
"allOf": [
{
"$ref": "#/components/schemas/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"
}
]
},
"spec": {
"description": "spec overrides the Pod template spec. These values will be merged with the TrainingRuntime's Pod template spec.",
"allOf": [
{
"$ref": "#/components/schemas/trainer.v1alpha1.PodTemplateSpecOverride"
}
]
},
"targetJobs": {
"description": "targetJobs is the list of replicated jobs in the training runtime template to apply the overrides.",
"type": "array",
"items": {
"default": {},
"allOf": [
{
"$ref": "#/components/schemas/trainer.v1alpha1.PodTemplateOverrideTargetJob"
}
]
},
"x-kubernetes-list-type": "atomic"
}
}
},
"trainer.v1alpha1.PodTemplateOverrideTargetJob": {
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"description": "name is the target training job name for which the PodTemplateSpec is overridden.",
"type": "string"
}
}
},
"trainer.v1alpha1.PodTemplateSpecOverride": {
"description": "PodTemplateSpecOverride represents the spec overrides for Pod template.",
"trainer.v1alpha1.PodSpecPatch": {
"description": "PodSpecPatch contains the Pod spec fields that managers are permitted to patch.",
"type": "object",
"properties": {
"affinity": {
"description": "affinity overrides for the Pod's affinity.",
"description": "affinity patches the Pod's scheduling affinity.",
"allOf": [
{
"$ref": "#/components/schemas/io.k8s.api.core.v1.Affinity"
@@ -14187,13 +14213,13 @@
]
},
"containers": {
"description": "containers overrides for the containers in the target job templates.",
"description": "containers patches specific containers in the target job templates.",
"type": "array",
"items": {
"default": {},
"allOf": [
{
"$ref": "#/components/schemas/trainer.v1alpha1.ContainerOverride"
"$ref": "#/components/schemas/trainer.v1alpha1.ContainerPatch"
}
]
},
@@ -14203,7 +14229,7 @@
"x-kubernetes-list-type": "map"
},
"imagePullSecrets": {
"description": "imagePullSecrets overrides the image pull secrets for the Pods in the target job templates.",
"description": "imagePullSecrets patches the image pull secrets for the Pods in the target job templates.",
"type": "array",
"items": {
"default": {},
@@ -14219,13 +14245,13 @@
"x-kubernetes-list-type": "map"
},
"initContainers": {
"description": "initContainers overrides the init container in the target job templates.",
"description": "initContainers patches the init containers in the target job templates.",
"type": "array",
"items": {
"default": {},
"allOf": [
{
"$ref": "#/components/schemas/trainer.v1alpha1.ContainerOverride"
"$ref": "#/components/schemas/trainer.v1alpha1.ContainerPatch"
}
]
},
@@ -14235,7 +14261,7 @@
"x-kubernetes-list-type": "map"
},
"nodeSelector": {
"description": "nodeSelector overrides the node selector to place Pod on the specific node.",
"description": "nodeSelector patches the node selector to place Pods on specific nodes.",
"type": "object",
"additionalProperties": {
"type": "string",
@@ -14243,7 +14269,7 @@
}
},
"schedulingGates": {
"description": "schedulingGates overrides the scheduling gates of the Pods in the target job templates. More info: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-scheduling-readiness/",
"description": "schedulingGates patches the scheduling gates for the Pods in the target job templates. More info: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-scheduling-readiness/",
"type": "array",
"items": {
"default": {},
@@ -14259,7 +14285,7 @@
"x-kubernetes-list-type": "map"
},
"securityContext": {
"description": "securityContext overrides the Pod's security context. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/",
"description": "securityContext patches the Pod's security context. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/",
"allOf": [
{
"$ref": "#/components/schemas/io.k8s.api.core.v1.PodSecurityContext"
@@ -14267,11 +14293,11 @@
]
},
"serviceAccountName": {
"description": "serviceAccountName overrides the service account.",
"description": "serviceAccountName patches the service account for the Pods in the target job templates.",
"type": "string"
},
"tolerations": {
"description": "tolerations overrides the Pod's tolerations.",
"description": "tolerations patches the Pod's tolerations.",
"type": "array",
"items": {
"default": {},
@@ -14284,7 +14310,7 @@
"x-kubernetes-list-type": "atomic"
},
"volumes": {
"description": "volumes overrides the Pod's volumes.",
"description": "volumes patches the Pod's volumes.",
"type": "array",
"items": {
"default": {},
@@ -14301,6 +14327,78 @@
}
}
},
"trainer.v1alpha1.PodTemplatePatch": {
"description": "PodTemplatePatch defines patches for a Pod template within a Job.",
"type": "object",
"properties": {
"metadata": {
"description": "metadata patches the Pod template metadata. Only labels and annotations are allowed.",
"allOf": [
{
"$ref": "#/components/schemas/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta"
}
]
},
"spec": {
"description": "spec patches the Pod spec with the fields managers are permitted to set.",
"allOf": [
{
"$ref": "#/components/schemas/trainer.v1alpha1.PodSpecPatch"
}
]
}
}
},
"trainer.v1alpha1.ReplicatedJobPatch": {
"description": "ReplicatedJobPatch defines patches for a specific replicated job within the JobSet.",
"type": "object",
"required": [
"name"
],
"properties": {
"name": {
"description": "name is the name of the replicated job to patch.",
"type": "string"
},
"template": {
"description": "template patches the Job template for this replicated job.",
"allOf": [
{
"$ref": "#/components/schemas/trainer.v1alpha1.JobTemplatePatch"
}
]
}
}
},
"trainer.v1alpha1.RuntimePatch": {
"description": "RuntimePatch represents a custom patch applied to the TrainJob's training runtime template. Patches are keyed by manager to provide clear ownership and avoid conflicts between controllers.",
"type": "object",
"required": [
"manager"
],
"properties": {
"manager": {
"description": "manager indicates who owns this patch entry. It can be set by the user, external controllers, or admission webhooks to track ownership and avoid conflicts. For example, Kueue sets this field to \"kueue.x-k8s.io/manager\".",
"type": "string"
},
"time": {
"description": "time is the timestamp of when this patch was last written. Set by the Trainer admission webhook on each create or update. Used for observability only, not used as a list map key.",
"allOf": [
{
"$ref": "#/components/schemas/io.k8s.apimachinery.pkg.apis.meta.v1.Time"
}
]
},
"trainingRuntimeSpec": {
"description": "trainingRuntimeSpec defines allowed patches for ClusterTrainingRuntime or TrainingRuntime-based jobs.",
"allOf": [
{
"$ref": "#/components/schemas/trainer.v1alpha1.TrainingRuntimeSpecPatch"
}
]
}
}
},
"trainer.v1alpha1.RuntimeRef": {
"description": "RuntimeRef represents the reference to the existing training runtime.",
"type": "object",
@@ -14412,14 +14510,6 @@
"runtimeRef"
],
"properties": {
"annotations": {
"description": "annotations to apply for the derivative JobSet and Jobs. They will be merged with the TrainingRuntime values.",
"type": "object",
"additionalProperties": {
"type": "string",
"default": ""
}
},
"initializer": {
"description": "initializer defines the configuration of the initializer.",
"allOf": [
@@ -14428,30 +14518,25 @@
}
]
},
"labels": {
"description": "labels to apply for the derivative JobSet and Jobs. They will be merged with the TrainingRuntime values.",
"type": "object",
"additionalProperties": {
"type": "string",
"default": ""
}
},
"managedBy": {
"description": "managedBy is used to indicate the controller or entity that manages a TrainJob. The value must be either an empty, `trainer.kubeflow.org/trainjob-controller` or `kueue.x-k8s.io/multikueue`. The built-in TrainJob controller reconciles TrainJob which don't have this field at all or the field value is the reserved string `trainer.kubeflow.org/trainjob-controller`, but delegates reconciling TrainJobs with a 'kueue.x-k8s.io/multikueue' to the Kueue. The field is immutable.",
"type": "string"
},
"podTemplateOverrides": {
"description": "podTemplateOverrides define the PodTemplateOverrides for the training runtime. When multiple overrides apply to the same targetJob, later entries in the slice override earlier field values.",
"runtimePatches": {
"description": "runtimePatches defines custom patches applied to the TrainJob's Runtime. Patches are keyed by manager to provide clear ownership and avoid conflicts between controllers.",
"type": "array",
"items": {
"default": {},
"allOf": [
{
"$ref": "#/components/schemas/trainer.v1alpha1.PodTemplateOverride"
"$ref": "#/components/schemas/trainer.v1alpha1.RuntimePatch"
}
]
},
"x-kubernetes-list-type": "atomic"
"x-kubernetes-list-map-keys": [
"manager"
],
"x-kubernetes-list-type": "map"
},
"runtimeRef": {
"description": "runtimeRef is the reference to the training runtime.",
@@ -14677,6 +14762,20 @@
}
}
},
"trainer.v1alpha1.TrainingRuntimeSpecPatch": {
"description": "TrainingRuntimeSpecPatch mirrors TrainingRuntimeSpec but only exposes the fields managers are permitted to patch.",
"type": "object",
"properties": {
"template": {
"description": "template patches the JobSet template.",
"allOf": [
{
"$ref": "#/components/schemas/trainer.v1alpha1.JobSetTemplatePatch"
}
]
}
}
},
"trainer.v1alpha1.VolcanoPodGroupPolicySource": {
"description": "VolcanoPodGroupPolicySource represents configuration for the Volcano gang-scheduler.",
"type": "object",
api/python_api/kubeflow_trainer_api/models/__init__.py
Generated
+10 -4
View File
@@ -358,22 +358,27 @@ from kubeflow_trainer_api.models.jobset_v1alpha2_volume_retention_policy import
from kubeflow_trainer_api.models.scheduling_v1beta1_network_topology_spec import SchedulingV1beta1NetworkTopologySpec
from kubeflow_trainer_api.models.trainer_v1alpha1_cluster_training_runtime import TrainerV1alpha1ClusterTrainingRuntime
from kubeflow_trainer_api.models.trainer_v1alpha1_cluster_training_runtime_list import TrainerV1alpha1ClusterTrainingRuntimeList
from kubeflow_trainer_api.models.trainer_v1alpha1_container_override import TrainerV1alpha1ContainerOverride
from kubeflow_trainer_api.models.trainer_v1alpha1_container_patch import TrainerV1alpha1ContainerPatch
from kubeflow_trainer_api.models.trainer_v1alpha1_coscheduling_pod_group_policy_source import TrainerV1alpha1CoschedulingPodGroupPolicySource
from kubeflow_trainer_api.models.trainer_v1alpha1_dataset_initializer import TrainerV1alpha1DatasetInitializer
from kubeflow_trainer_api.models.trainer_v1alpha1_flux_ml_policy_source import TrainerV1alpha1FluxMLPolicySource
from kubeflow_trainer_api.models.trainer_v1alpha1_initializer import TrainerV1alpha1Initializer
from kubeflow_trainer_api.models.trainer_v1alpha1_job_set_spec_patch import TrainerV1alpha1JobSetSpecPatch
from kubeflow_trainer_api.models.trainer_v1alpha1_job_set_template_patch import TrainerV1alpha1JobSetTemplatePatch
from kubeflow_trainer_api.models.trainer_v1alpha1_job_set_template_spec import TrainerV1alpha1JobSetTemplateSpec
from kubeflow_trainer_api.models.trainer_v1alpha1_job_spec_patch import TrainerV1alpha1JobSpecPatch
from kubeflow_trainer_api.models.trainer_v1alpha1_job_status import TrainerV1alpha1JobStatus
from kubeflow_trainer_api.models.trainer_v1alpha1_job_template_patch import TrainerV1alpha1JobTemplatePatch
from kubeflow_trainer_api.models.trainer_v1alpha1_ml_policy import TrainerV1alpha1MLPolicy
from kubeflow_trainer_api.models.trainer_v1alpha1_ml_policy_source import TrainerV1alpha1MLPolicySource
from kubeflow_trainer_api.models.trainer_v1alpha1_mpiml_policy_source import TrainerV1alpha1MPIMLPolicySource
from kubeflow_trainer_api.models.trainer_v1alpha1_model_initializer import TrainerV1alpha1ModelInitializer
from kubeflow_trainer_api.models.trainer_v1alpha1_pod_group_policy import TrainerV1alpha1PodGroupPolicy
from kubeflow_trainer_api.models.trainer_v1alpha1_pod_group_policy_source import TrainerV1alpha1PodGroupPolicySource
from kubeflow_trainer_api.models.trainer_v1alpha1_pod_template_override import TrainerV1alpha1PodTemplateOverride
from kubeflow_trainer_api.models.trainer_v1alpha1_pod_template_override_target_job import TrainerV1alpha1PodTemplateOverrideTargetJob
from kubeflow_trainer_api.models.trainer_v1alpha1_pod_template_spec_override import TrainerV1alpha1PodTemplateSpecOverride
from kubeflow_trainer_api.models.trainer_v1alpha1_pod_spec_patch import TrainerV1alpha1PodSpecPatch
from kubeflow_trainer_api.models.trainer_v1alpha1_pod_template_patch import TrainerV1alpha1PodTemplatePatch
from kubeflow_trainer_api.models.trainer_v1alpha1_replicated_job_patch import TrainerV1alpha1ReplicatedJobPatch
from kubeflow_trainer_api.models.trainer_v1alpha1_runtime_patch import TrainerV1alpha1RuntimePatch
from kubeflow_trainer_api.models.trainer_v1alpha1_runtime_ref import TrainerV1alpha1RuntimeRef
from kubeflow_trainer_api.models.trainer_v1alpha1_train_job import TrainerV1alpha1TrainJob
from kubeflow_trainer_api.models.trainer_v1alpha1_train_job_list import TrainerV1alpha1TrainJobList
@@ -383,4 +388,5 @@ from kubeflow_trainer_api.models.trainer_v1alpha1_trainer import TrainerV1alpha1
from kubeflow_trainer_api.models.trainer_v1alpha1_training_runtime import TrainerV1alpha1TrainingRuntime
from kubeflow_trainer_api.models.trainer_v1alpha1_training_runtime_list import TrainerV1alpha1TrainingRuntimeList
from kubeflow_trainer_api.models.trainer_v1alpha1_training_runtime_spec import TrainerV1alpha1TrainingRuntimeSpec
from kubeflow_trainer_api.models.trainer_v1alpha1_training_runtime_spec_patch import TrainerV1alpha1TrainingRuntimeSpecPatch
from kubeflow_trainer_api.models.trainer_v1alpha1_volcano_pod_group_policy_source import TrainerV1alpha1VolcanoPodGroupPolicySource
api/python_api/kubeflow_trainer_api/models/trainer_v1alpha1_container_patch.py
Generated
+113
View File
@@ -0,0 +1,113 @@
# coding: utf-8
"""
Kubeflow Trainer OpenAPI Spec
No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
The version of the OpenAPI document: unversioned
Generated by OpenAPI Generator (https://openapi-generator.tech)
Do not edit the class manually.
""" # noqa: E501
from __future__ import annotations
import pprint
import re # noqa: F401
import json
from pydantic import BaseModel, ConfigDict, Field, StrictStr
from typing import Any, ClassVar, Dict, List, Optional
from kubeflow_trainer_api.models.io_k8s_api_core_v1_env_var import IoK8sApiCoreV1EnvVar
from kubeflow_trainer_api.models.io_k8s_api_core_v1_security_context import IoK8sApiCoreV1SecurityContext
from kubeflow_trainer_api.models.io_k8s_api_core_v1_volume_mount import IoK8sApiCoreV1VolumeMount
from typing import Optional, Set
from typing_extensions import Self
class TrainerV1alpha1ContainerPatch(BaseModel):
"""
ContainerPatch represents parameters that can be patched using PodSpecPatch.
""" # noqa: E501
env: Optional[List[IoK8sApiCoreV1EnvVar]] = Field(default=None, description="env is the list of environment variables to set in the container. These values will be merged with the Runtime's environments. These values can't be set for container with the name: `node`, `dataset-initializer`, or `model-initializer`. For those containers the envs can only be set via Trainer or Initializer APIs.")
name: StrictStr = Field(description="name for the container. Runtime must have this container.")
security_context: Optional[IoK8sApiCoreV1SecurityContext] = Field(default=None, description="securityContext patches the container's security context. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/", alias="securityContext")
volume_mounts: Optional[List[IoK8sApiCoreV1VolumeMount]] = Field(default=None, description="volumeMounts are the volumes to mount into the container's filesystem.", alias="volumeMounts")
__properties: ClassVar[List[str]] = ["env", "name", "securityContext", "volumeMounts"]
model_config = ConfigDict(
populate_by_name=True,
validate_assignment=True,
protected_namespaces=(),
)
def to_str(self) -> str:
"""Returns the string representation of the model using alias"""
return pprint.pformat(self.model_dump(by_alias=True))
def to_json(self) -> str:
"""Returns the JSON representation of the model using alias"""
# TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
return json.dumps(self.to_dict())
@classmethod
def from_json(cls, json_str: str) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1ContainerPatch from a JSON string"""
return cls.from_dict(json.loads(json_str))
def to_dict(self) -> Dict[str, Any]:
"""Return the dictionary representation of the model using alias.
This has the following differences from calling pydantic's
`self.model_dump(by_alias=True)`:
* `None` is only added to the output dict for nullable fields that
were set at model initialization. Other fields with value `None`
are ignored.
"""
excluded_fields: Set[str] = set([
])
_dict = self.model_dump(
by_alias=True,
exclude=excluded_fields,
exclude_none=True,
)
# override the default output from pydantic by calling `to_dict()` of each item in env (list)
_items = []
if self.env:
for _item_env in self.env:
if _item_env:
_items.append(_item_env.to_dict())
_dict['env'] = _items
# override the default output from pydantic by calling `to_dict()` of security_context
if self.security_context:
_dict['securityContext'] = self.security_context.to_dict()
# override the default output from pydantic by calling `to_dict()` of each item in volume_mounts (list)
_items = []
if self.volume_mounts:
for _item_volume_mounts in self.volume_mounts:
if _item_volume_mounts:
_items.append(_item_volume_mounts.to_dict())
_dict['volumeMounts'] = _items
return _dict
@classmethod
def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1ContainerPatch from a dict"""
if obj is None:
return None
if not isinstance(obj, dict):
return cls.model_validate(obj)
_obj = cls.model_validate({
"env": [IoK8sApiCoreV1EnvVar.from_dict(_item) for _item in obj["env"]] if obj.get("env") is not None else None,
"name": obj.get("name"),
"securityContext": IoK8sApiCoreV1SecurityContext.from_dict(obj["securityContext"]) if obj.get("securityContext") is not None else None,
"volumeMounts": [IoK8sApiCoreV1VolumeMount.from_dict(_item) for _item in obj["volumeMounts"]] if obj.get("volumeMounts") is not None else None
})
return _obj
api/python_api/kubeflow_trainer_api/models/trainer_v1alpha1_job_set_spec_patch.py
Generated
+95
View File
@@ -0,0 +1,95 @@
# coding: utf-8
"""
Kubeflow Trainer OpenAPI Spec
No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
The version of the OpenAPI document: unversioned
Generated by OpenAPI Generator (https://openapi-generator.tech)
Do not edit the class manually.
""" # noqa: E501
from __future__ import annotations
import pprint
import re # noqa: F401
import json
from pydantic import BaseModel, ConfigDict, Field
from typing import Any, ClassVar, Dict, List, Optional
from kubeflow_trainer_api.models.trainer_v1alpha1_replicated_job_patch import TrainerV1alpha1ReplicatedJobPatch
from typing import Optional, Set
from typing_extensions import Self
class TrainerV1alpha1JobSetSpecPatch(BaseModel):
"""
JobSetSpecPatch defines allowed patches for the JobSet spec.
""" # noqa: E501
replicated_jobs: Optional[List[TrainerV1alpha1ReplicatedJobPatch]] = Field(default=None, description="replicatedJobs defines per-job patches, keyed by job name.", alias="replicatedJobs")
__properties: ClassVar[List[str]] = ["replicatedJobs"]
model_config = ConfigDict(
populate_by_name=True,
validate_assignment=True,
protected_namespaces=(),
)
def to_str(self) -> str:
"""Returns the string representation of the model using alias"""
return pprint.pformat(self.model_dump(by_alias=True))
def to_json(self) -> str:
"""Returns the JSON representation of the model using alias"""
# TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
return json.dumps(self.to_dict())
@classmethod
def from_json(cls, json_str: str) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1JobSetSpecPatch from a JSON string"""
return cls.from_dict(json.loads(json_str))
def to_dict(self) -> Dict[str, Any]:
"""Return the dictionary representation of the model using alias.
This has the following differences from calling pydantic's
`self.model_dump(by_alias=True)`:
* `None` is only added to the output dict for nullable fields that
were set at model initialization. Other fields with value `None`
are ignored.
"""
excluded_fields: Set[str] = set([
])
_dict = self.model_dump(
by_alias=True,
exclude=excluded_fields,
exclude_none=True,
)
# override the default output from pydantic by calling `to_dict()` of each item in replicated_jobs (list)
_items = []
if self.replicated_jobs:
for _item_replicated_jobs in self.replicated_jobs:
if _item_replicated_jobs:
_items.append(_item_replicated_jobs.to_dict())
_dict['replicatedJobs'] = _items
return _dict
@classmethod
def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1JobSetSpecPatch from a dict"""
if obj is None:
return None
if not isinstance(obj, dict):
return cls.model_validate(obj)
_obj = cls.model_validate({
"replicatedJobs": [TrainerV1alpha1ReplicatedJobPatch.from_dict(_item) for _item in obj["replicatedJobs"]] if obj.get("replicatedJobs") is not None else None
})
return _obj
api/python_api/kubeflow_trainer_api/models/trainer_v1alpha1_job_set_template_patch.py
Generated
+97
View File
@@ -0,0 +1,97 @@
# coding: utf-8
"""
Kubeflow Trainer OpenAPI Spec
No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
The version of the OpenAPI document: unversioned
Generated by OpenAPI Generator (https://openapi-generator.tech)
Do not edit the class manually.
""" # noqa: E501
from __future__ import annotations
import pprint
import re # noqa: F401
import json
from pydantic import BaseModel, ConfigDict, Field
from typing import Any, ClassVar, Dict, List, Optional
from kubeflow_trainer_api.models.io_k8s_apimachinery_pkg_apis_meta_v1_object_meta import IoK8sApimachineryPkgApisMetaV1ObjectMeta
from kubeflow_trainer_api.models.trainer_v1alpha1_job_set_spec_patch import TrainerV1alpha1JobSetSpecPatch
from typing import Optional, Set
from typing_extensions import Self
class TrainerV1alpha1JobSetTemplatePatch(BaseModel):
"""
JobSetTemplatePatch defines patches for the JobSet template. It mirrors JobSetTemplateSpec but only exposes metadata and restricted spec fields.
""" # noqa: E501
metadata: Optional[IoK8sApimachineryPkgApisMetaV1ObjectMeta] = Field(default=None, description="metadata patches the JobSet object metadata. Only labels and annotations are allowed.")
spec: Optional[TrainerV1alpha1JobSetSpecPatch] = Field(default=None, description="spec patches the JobSet spec with restricted fields.")
__properties: ClassVar[List[str]] = ["metadata", "spec"]
model_config = ConfigDict(
populate_by_name=True,
validate_assignment=True,
protected_namespaces=(),
)
def to_str(self) -> str:
"""Returns the string representation of the model using alias"""
return pprint.pformat(self.model_dump(by_alias=True))
def to_json(self) -> str:
"""Returns the JSON representation of the model using alias"""
# TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
return json.dumps(self.to_dict())
@classmethod
def from_json(cls, json_str: str) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1JobSetTemplatePatch from a JSON string"""
return cls.from_dict(json.loads(json_str))
def to_dict(self) -> Dict[str, Any]:
"""Return the dictionary representation of the model using alias.
This has the following differences from calling pydantic's
`self.model_dump(by_alias=True)`:
* `None` is only added to the output dict for nullable fields that
were set at model initialization. Other fields with value `None`
are ignored.
"""
excluded_fields: Set[str] = set([
])
_dict = self.model_dump(
by_alias=True,
exclude=excluded_fields,
exclude_none=True,
)
# override the default output from pydantic by calling `to_dict()` of metadata
if self.metadata:
_dict['metadata'] = self.metadata.to_dict()
# override the default output from pydantic by calling `to_dict()` of spec
if self.spec:
_dict['spec'] = self.spec.to_dict()
return _dict
@classmethod
def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1JobSetTemplatePatch from a dict"""
if obj is None:
return None
if not isinstance(obj, dict):
return cls.model_validate(obj)
_obj = cls.model_validate({
"metadata": IoK8sApimachineryPkgApisMetaV1ObjectMeta.from_dict(obj["metadata"]) if obj.get("metadata") is not None else None,
"spec": TrainerV1alpha1JobSetSpecPatch.from_dict(obj["spec"]) if obj.get("spec") is not None else None
})
return _obj
api/python_api/kubeflow_trainer_api/models/trainer_v1alpha1_job_spec_patch.py
Generated
+91
View File
@@ -0,0 +1,91 @@
# coding: utf-8
"""
Kubeflow Trainer OpenAPI Spec
No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
The version of the OpenAPI document: unversioned
Generated by OpenAPI Generator (https://openapi-generator.tech)
Do not edit the class manually.
""" # noqa: E501
from __future__ import annotations
import pprint
import re # noqa: F401
import json
from pydantic import BaseModel, ConfigDict, Field
from typing import Any, ClassVar, Dict, List, Optional
from kubeflow_trainer_api.models.trainer_v1alpha1_pod_template_patch import TrainerV1alpha1PodTemplatePatch
from typing import Optional, Set
from typing_extensions import Self
class TrainerV1alpha1JobSpecPatch(BaseModel):
"""
JobSpecPatch defines allowed patches for the Job spec.
""" # noqa: E501
template: Optional[TrainerV1alpha1PodTemplatePatch] = Field(default=None, description="template patches the Pod template for this Job.")
__properties: ClassVar[List[str]] = ["template"]
model_config = ConfigDict(
populate_by_name=True,
validate_assignment=True,
protected_namespaces=(),
)
def to_str(self) -> str:
"""Returns the string representation of the model using alias"""
return pprint.pformat(self.model_dump(by_alias=True))
def to_json(self) -> str:
"""Returns the JSON representation of the model using alias"""
# TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
return json.dumps(self.to_dict())
@classmethod
def from_json(cls, json_str: str) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1JobSpecPatch from a JSON string"""
return cls.from_dict(json.loads(json_str))
def to_dict(self) -> Dict[str, Any]:
"""Return the dictionary representation of the model using alias.
This has the following differences from calling pydantic's
`self.model_dump(by_alias=True)`:
* `None` is only added to the output dict for nullable fields that
were set at model initialization. Other fields with value `None`
are ignored.
"""
excluded_fields: Set[str] = set([
])
_dict = self.model_dump(
by_alias=True,
exclude=excluded_fields,
exclude_none=True,
)
# override the default output from pydantic by calling `to_dict()` of template
if self.template:
_dict['template'] = self.template.to_dict()
return _dict
@classmethod
def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1JobSpecPatch from a dict"""
if obj is None:
return None
if not isinstance(obj, dict):
return cls.model_validate(obj)
_obj = cls.model_validate({
"template": TrainerV1alpha1PodTemplatePatch.from_dict(obj["template"]) if obj.get("template") is not None else None
})
return _obj
api/python_api/kubeflow_trainer_api/models/trainer_v1alpha1_job_template_patch.py
Generated
+97
View File
@@ -0,0 +1,97 @@
# coding: utf-8
"""
Kubeflow Trainer OpenAPI Spec
No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
The version of the OpenAPI document: unversioned
Generated by OpenAPI Generator (https://openapi-generator.tech)
Do not edit the class manually.
""" # noqa: E501
from __future__ import annotations
import pprint
import re # noqa: F401
import json
from pydantic import BaseModel, ConfigDict, Field
from typing import Any, ClassVar, Dict, List, Optional
from kubeflow_trainer_api.models.io_k8s_apimachinery_pkg_apis_meta_v1_object_meta import IoK8sApimachineryPkgApisMetaV1ObjectMeta
from kubeflow_trainer_api.models.trainer_v1alpha1_job_spec_patch import TrainerV1alpha1JobSpecPatch
from typing import Optional, Set
from typing_extensions import Self
class TrainerV1alpha1JobTemplatePatch(BaseModel):
"""
JobTemplatePatch defines patches for a Job template within a replicated job.
""" # noqa: E501
metadata: Optional[IoK8sApimachineryPkgApisMetaV1ObjectMeta] = Field(default=None, description="metadata patches the Job template metadata. Only labels and annotations are allowed.")
spec: Optional[TrainerV1alpha1JobSpecPatch] = Field(default=None, description="spec patches the Job spec with restricted fields.")
__properties: ClassVar[List[str]] = ["metadata", "spec"]
model_config = ConfigDict(
populate_by_name=True,
validate_assignment=True,
protected_namespaces=(),
)
def to_str(self) -> str:
"""Returns the string representation of the model using alias"""
return pprint.pformat(self.model_dump(by_alias=True))
def to_json(self) -> str:
"""Returns the JSON representation of the model using alias"""
# TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
return json.dumps(self.to_dict())
@classmethod
def from_json(cls, json_str: str) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1JobTemplatePatch from a JSON string"""
return cls.from_dict(json.loads(json_str))
def to_dict(self) -> Dict[str, Any]:
"""Return the dictionary representation of the model using alias.
This has the following differences from calling pydantic's
`self.model_dump(by_alias=True)`:
* `None` is only added to the output dict for nullable fields that
were set at model initialization. Other fields with value `None`
are ignored.
"""
excluded_fields: Set[str] = set([
])
_dict = self.model_dump(
by_alias=True,
exclude=excluded_fields,
exclude_none=True,
)
# override the default output from pydantic by calling `to_dict()` of metadata
if self.metadata:
_dict['metadata'] = self.metadata.to_dict()
# override the default output from pydantic by calling `to_dict()` of spec
if self.spec:
_dict['spec'] = self.spec.to_dict()
return _dict
@classmethod
def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1JobTemplatePatch from a dict"""
if obj is None:
return None
if not isinstance(obj, dict):
return cls.model_validate(obj)
_obj = cls.model_validate({
"metadata": IoK8sApimachineryPkgApisMetaV1ObjectMeta.from_dict(obj["metadata"]) if obj.get("metadata") is not None else None,
"spec": TrainerV1alpha1JobSpecPatch.from_dict(obj["spec"]) if obj.get("spec") is not None else None
})
return _obj
api/python_api/kubeflow_trainer_api/models/trainer_v1alpha1_pod_spec_patch.py
Generated
+160
View File
@@ -0,0 +1,160 @@
# coding: utf-8
"""
Kubeflow Trainer OpenAPI Spec
No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
The version of the OpenAPI document: unversioned
Generated by OpenAPI Generator (https://openapi-generator.tech)
Do not edit the class manually.
""" # noqa: E501
from __future__ import annotations
import pprint
import re # noqa: F401
import json
from pydantic import BaseModel, ConfigDict, Field, StrictStr
from typing import Any, ClassVar, Dict, List, Optional
from kubeflow_trainer_api.models.io_k8s_api_core_v1_affinity import IoK8sApiCoreV1Affinity
from kubeflow_trainer_api.models.io_k8s_api_core_v1_local_object_reference import IoK8sApiCoreV1LocalObjectReference
from kubeflow_trainer_api.models.io_k8s_api_core_v1_pod_scheduling_gate import IoK8sApiCoreV1PodSchedulingGate
from kubeflow_trainer_api.models.io_k8s_api_core_v1_pod_security_context import IoK8sApiCoreV1PodSecurityContext
from kubeflow_trainer_api.models.io_k8s_api_core_v1_toleration import IoK8sApiCoreV1Toleration
from kubeflow_trainer_api.models.io_k8s_api_core_v1_volume import IoK8sApiCoreV1Volume
from kubeflow_trainer_api.models.trainer_v1alpha1_container_patch import TrainerV1alpha1ContainerPatch
from typing import Optional, Set
from typing_extensions import Self
class TrainerV1alpha1PodSpecPatch(BaseModel):
"""
PodSpecPatch contains the Pod spec fields that managers are permitted to patch.
""" # noqa: E501
affinity: Optional[IoK8sApiCoreV1Affinity] = Field(default=None, description="affinity patches the Pod's scheduling affinity.")
containers: Optional[List[TrainerV1alpha1ContainerPatch]] = Field(default=None, description="containers patches specific containers in the target job templates.")
image_pull_secrets: Optional[List[IoK8sApiCoreV1LocalObjectReference]] = Field(default=None, description="imagePullSecrets patches the image pull secrets for the Pods in the target job templates.", alias="imagePullSecrets")
init_containers: Optional[List[TrainerV1alpha1ContainerPatch]] = Field(default=None, description="initContainers patches the init containers in the target job templates.", alias="initContainers")
node_selector: Optional[Dict[str, StrictStr]] = Field(default=None, description="nodeSelector patches the node selector to place Pods on specific nodes.", alias="nodeSelector")
scheduling_gates: Optional[List[IoK8sApiCoreV1PodSchedulingGate]] = Field(default=None, description="schedulingGates patches the scheduling gates for the Pods in the target job templates. More info: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-scheduling-readiness/", alias="schedulingGates")
security_context: Optional[IoK8sApiCoreV1PodSecurityContext] = Field(default=None, description="securityContext patches the Pod's security context. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/", alias="securityContext")
service_account_name: Optional[StrictStr] = Field(default=None, description="serviceAccountName patches the service account for the Pods in the target job templates.", alias="serviceAccountName")
tolerations: Optional[List[IoK8sApiCoreV1Toleration]] = Field(default=None, description="tolerations patches the Pod's tolerations.")
volumes: Optional[List[IoK8sApiCoreV1Volume]] = Field(default=None, description="volumes patches the Pod's volumes.")
__properties: ClassVar[List[str]] = ["affinity", "containers", "imagePullSecrets", "initContainers", "nodeSelector", "schedulingGates", "securityContext", "serviceAccountName", "tolerations", "volumes"]
model_config = ConfigDict(
populate_by_name=True,
validate_assignment=True,
protected_namespaces=(),
)
def to_str(self) -> str:
"""Returns the string representation of the model using alias"""
return pprint.pformat(self.model_dump(by_alias=True))
def to_json(self) -> str:
"""Returns the JSON representation of the model using alias"""
# TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
return json.dumps(self.to_dict())
@classmethod
def from_json(cls, json_str: str) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1PodSpecPatch from a JSON string"""
return cls.from_dict(json.loads(json_str))
def to_dict(self) -> Dict[str, Any]:
"""Return the dictionary representation of the model using alias.
This has the following differences from calling pydantic's
`self.model_dump(by_alias=True)`:
* `None` is only added to the output dict for nullable fields that
were set at model initialization. Other fields with value `None`
are ignored.
"""
excluded_fields: Set[str] = set([
])
_dict = self.model_dump(
by_alias=True,
exclude=excluded_fields,
exclude_none=True,
)
# override the default output from pydantic by calling `to_dict()` of affinity
if self.affinity:
_dict['affinity'] = self.affinity.to_dict()
# override the default output from pydantic by calling `to_dict()` of each item in containers (list)
_items = []
if self.containers:
for _item_containers in self.containers:
if _item_containers:
_items.append(_item_containers.to_dict())
_dict['containers'] = _items
# override the default output from pydantic by calling `to_dict()` of each item in image_pull_secrets (list)
_items = []
if self.image_pull_secrets:
for _item_image_pull_secrets in self.image_pull_secrets:
if _item_image_pull_secrets:
_items.append(_item_image_pull_secrets.to_dict())
_dict['imagePullSecrets'] = _items
# override the default output from pydantic by calling `to_dict()` of each item in init_containers (list)
_items = []
if self.init_containers:
for _item_init_containers in self.init_containers:
if _item_init_containers:
_items.append(_item_init_containers.to_dict())
_dict['initContainers'] = _items
# override the default output from pydantic by calling `to_dict()` of each item in scheduling_gates (list)
_items = []
if self.scheduling_gates:
for _item_scheduling_gates in self.scheduling_gates:
if _item_scheduling_gates:
_items.append(_item_scheduling_gates.to_dict())
_dict['schedulingGates'] = _items
# override the default output from pydantic by calling `to_dict()` of security_context
if self.security_context:
_dict['securityContext'] = self.security_context.to_dict()
# override the default output from pydantic by calling `to_dict()` of each item in tolerations (list)
_items = []
if self.tolerations:
for _item_tolerations in self.tolerations:
if _item_tolerations:
_items.append(_item_tolerations.to_dict())
_dict['tolerations'] = _items
# override the default output from pydantic by calling `to_dict()` of each item in volumes (list)
_items = []
if self.volumes:
for _item_volumes in self.volumes:
if _item_volumes:
_items.append(_item_volumes.to_dict())
_dict['volumes'] = _items
return _dict
@classmethod
def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1PodSpecPatch from a dict"""
if obj is None:
return None
if not isinstance(obj, dict):
return cls.model_validate(obj)
_obj = cls.model_validate({
"affinity": IoK8sApiCoreV1Affinity.from_dict(obj["affinity"]) if obj.get("affinity") is not None else None,
"containers": [TrainerV1alpha1ContainerPatch.from_dict(_item) for _item in obj["containers"]] if obj.get("containers") is not None else None,
"imagePullSecrets": [IoK8sApiCoreV1LocalObjectReference.from_dict(_item) for _item in obj["imagePullSecrets"]] if obj.get("imagePullSecrets") is not None else None,
"initContainers": [TrainerV1alpha1ContainerPatch.from_dict(_item) for _item in obj["initContainers"]] if obj.get("initContainers") is not None else None,
"nodeSelector": obj.get("nodeSelector"),
"schedulingGates": [IoK8sApiCoreV1PodSchedulingGate.from_dict(_item) for _item in obj["schedulingGates"]] if obj.get("schedulingGates") is not None else None,
"securityContext": IoK8sApiCoreV1PodSecurityContext.from_dict(obj["securityContext"]) if obj.get("securityContext") is not None else None,
"serviceAccountName": obj.get("serviceAccountName"),
"tolerations": [IoK8sApiCoreV1Toleration.from_dict(_item) for _item in obj["tolerations"]] if obj.get("tolerations") is not None else None,
"volumes": [IoK8sApiCoreV1Volume.from_dict(_item) for _item in obj["volumes"]] if obj.get("volumes") is not None else None
})
return _obj
api/python_api/kubeflow_trainer_api/models/trainer_v1alpha1_pod_template_patch.py
Generated
+97
View File
@@ -0,0 +1,97 @@
# coding: utf-8
"""
Kubeflow Trainer OpenAPI Spec
No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
The version of the OpenAPI document: unversioned
Generated by OpenAPI Generator (https://openapi-generator.tech)
Do not edit the class manually.
""" # noqa: E501
from __future__ import annotations
import pprint
import re # noqa: F401
import json
from pydantic import BaseModel, ConfigDict, Field
from typing import Any, ClassVar, Dict, List, Optional
from kubeflow_trainer_api.models.io_k8s_apimachinery_pkg_apis_meta_v1_object_meta import IoK8sApimachineryPkgApisMetaV1ObjectMeta
from kubeflow_trainer_api.models.trainer_v1alpha1_pod_spec_patch import TrainerV1alpha1PodSpecPatch
from typing import Optional, Set
from typing_extensions import Self
class TrainerV1alpha1PodTemplatePatch(BaseModel):
"""
PodTemplatePatch defines patches for a Pod template within a Job.
""" # noqa: E501
metadata: Optional[IoK8sApimachineryPkgApisMetaV1ObjectMeta] = Field(default=None, description="metadata patches the Pod template metadata. Only labels and annotations are allowed.")
spec: Optional[TrainerV1alpha1PodSpecPatch] = Field(default=None, description="spec patches the Pod spec with the fields managers are permitted to set.")
__properties: ClassVar[List[str]] = ["metadata", "spec"]
model_config = ConfigDict(
populate_by_name=True,
validate_assignment=True,
protected_namespaces=(),
)
def to_str(self) -> str:
"""Returns the string representation of the model using alias"""
return pprint.pformat(self.model_dump(by_alias=True))
def to_json(self) -> str:
"""Returns the JSON representation of the model using alias"""
# TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
return json.dumps(self.to_dict())
@classmethod
def from_json(cls, json_str: str) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1PodTemplatePatch from a JSON string"""
return cls.from_dict(json.loads(json_str))
def to_dict(self) -> Dict[str, Any]:
"""Return the dictionary representation of the model using alias.
This has the following differences from calling pydantic's
`self.model_dump(by_alias=True)`:
* `None` is only added to the output dict for nullable fields that
were set at model initialization. Other fields with value `None`
are ignored.
"""
excluded_fields: Set[str] = set([
])
_dict = self.model_dump(
by_alias=True,
exclude=excluded_fields,
exclude_none=True,
)
# override the default output from pydantic by calling `to_dict()` of metadata
if self.metadata:
_dict['metadata'] = self.metadata.to_dict()
# override the default output from pydantic by calling `to_dict()` of spec
if self.spec:
_dict['spec'] = self.spec.to_dict()
return _dict
@classmethod
def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1PodTemplatePatch from a dict"""
if obj is None:
return None
if not isinstance(obj, dict):
return cls.model_validate(obj)
_obj = cls.model_validate({
"metadata": IoK8sApimachineryPkgApisMetaV1ObjectMeta.from_dict(obj["metadata"]) if obj.get("metadata") is not None else None,
"spec": TrainerV1alpha1PodSpecPatch.from_dict(obj["spec"]) if obj.get("spec") is not None else None
})
return _obj
api/python_api/kubeflow_trainer_api/models/trainer_v1alpha1_replicated_job_patch.py
Generated
+93
View File
@@ -0,0 +1,93 @@
# coding: utf-8
"""
Kubeflow Trainer OpenAPI Spec
No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
The version of the OpenAPI document: unversioned
Generated by OpenAPI Generator (https://openapi-generator.tech)
Do not edit the class manually.
""" # noqa: E501
from __future__ import annotations
import pprint
import re # noqa: F401
import json
from pydantic import BaseModel, ConfigDict, Field, StrictStr
from typing import Any, ClassVar, Dict, List, Optional
from kubeflow_trainer_api.models.trainer_v1alpha1_job_template_patch import TrainerV1alpha1JobTemplatePatch
from typing import Optional, Set
from typing_extensions import Self
class TrainerV1alpha1ReplicatedJobPatch(BaseModel):
"""
ReplicatedJobPatch defines patches for a specific replicated job within the JobSet.
""" # noqa: E501
name: StrictStr = Field(description="name is the name of the replicated job to patch.")
template: Optional[TrainerV1alpha1JobTemplatePatch] = Field(default=None, description="template patches the Job template for this replicated job.")
__properties: ClassVar[List[str]] = ["name", "template"]
model_config = ConfigDict(
populate_by_name=True,
validate_assignment=True,
protected_namespaces=(),
)
def to_str(self) -> str:
"""Returns the string representation of the model using alias"""
return pprint.pformat(self.model_dump(by_alias=True))
def to_json(self) -> str:
"""Returns the JSON representation of the model using alias"""
# TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
return json.dumps(self.to_dict())
@classmethod
def from_json(cls, json_str: str) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1ReplicatedJobPatch from a JSON string"""
return cls.from_dict(json.loads(json_str))
def to_dict(self) -> Dict[str, Any]:
"""Return the dictionary representation of the model using alias.
This has the following differences from calling pydantic's
`self.model_dump(by_alias=True)`:
* `None` is only added to the output dict for nullable fields that
were set at model initialization. Other fields with value `None`
are ignored.
"""
excluded_fields: Set[str] = set([
])
_dict = self.model_dump(
by_alias=True,
exclude=excluded_fields,
exclude_none=True,
)
# override the default output from pydantic by calling `to_dict()` of template
if self.template:
_dict['template'] = self.template.to_dict()
return _dict
@classmethod
def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1ReplicatedJobPatch from a dict"""
if obj is None:
return None
if not isinstance(obj, dict):
return cls.model_validate(obj)
_obj = cls.model_validate({
"name": obj.get("name"),
"template": TrainerV1alpha1JobTemplatePatch.from_dict(obj["template"]) if obj.get("template") is not None else None
})
return _obj
api/python_api/kubeflow_trainer_api/models/trainer_v1alpha1_runtime_patch.py
Generated
+96
View File
@@ -0,0 +1,96 @@
# coding: utf-8
"""
Kubeflow Trainer OpenAPI Spec
No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
The version of the OpenAPI document: unversioned
Generated by OpenAPI Generator (https://openapi-generator.tech)
Do not edit the class manually.
""" # noqa: E501
from __future__ import annotations
import pprint
import re # noqa: F401
import json
from datetime import datetime
from pydantic import BaseModel, ConfigDict, Field, StrictStr
from typing import Any, ClassVar, Dict, List, Optional
from kubeflow_trainer_api.models.trainer_v1alpha1_training_runtime_spec_patch import TrainerV1alpha1TrainingRuntimeSpecPatch
from typing import Optional, Set
from typing_extensions import Self
class TrainerV1alpha1RuntimePatch(BaseModel):
"""
RuntimePatch represents a custom patch applied to the TrainJob's training runtime template. Patches are keyed by manager to provide clear ownership and avoid conflicts between controllers.
""" # noqa: E501
manager: StrictStr = Field(description="manager indicates who owns this patch entry. It can be set by the user, external controllers, or admission webhooks to track ownership and avoid conflicts. For example, Kueue sets this field to \"kueue.x-k8s.io/manager\".")
time: Optional[datetime] = Field(default=None, description="time is the timestamp of when this patch was last written. Set by the Trainer admission webhook on each create or update. Used for observability only, not used as a list map key.")
training_runtime_spec: Optional[TrainerV1alpha1TrainingRuntimeSpecPatch] = Field(default=None, description="trainingRuntimeSpec defines allowed patches for ClusterTrainingRuntime or TrainingRuntime-based jobs.", alias="trainingRuntimeSpec")
__properties: ClassVar[List[str]] = ["manager", "time", "trainingRuntimeSpec"]
model_config = ConfigDict(
populate_by_name=True,
validate_assignment=True,
protected_namespaces=(),
)
def to_str(self) -> str:
"""Returns the string representation of the model using alias"""
return pprint.pformat(self.model_dump(by_alias=True))
def to_json(self) -> str:
"""Returns the JSON representation of the model using alias"""
# TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
return json.dumps(self.to_dict())
@classmethod
def from_json(cls, json_str: str) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1RuntimePatch from a JSON string"""
return cls.from_dict(json.loads(json_str))
def to_dict(self) -> Dict[str, Any]:
"""Return the dictionary representation of the model using alias.
This has the following differences from calling pydantic's
`self.model_dump(by_alias=True)`:
* `None` is only added to the output dict for nullable fields that
were set at model initialization. Other fields with value `None`
are ignored.
"""
excluded_fields: Set[str] = set([
])
_dict = self.model_dump(
by_alias=True,
exclude=excluded_fields,
exclude_none=True,
)
# override the default output from pydantic by calling `to_dict()` of training_runtime_spec
if self.training_runtime_spec:
_dict['trainingRuntimeSpec'] = self.training_runtime_spec.to_dict()
return _dict
@classmethod
def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1RuntimePatch from a dict"""
if obj is None:
return None
if not isinstance(obj, dict):
return cls.model_validate(obj)
_obj = cls.model_validate({
"manager": obj.get("manager"),
"time": obj.get("time"),
"trainingRuntimeSpec": TrainerV1alpha1TrainingRuntimeSpecPatch.from_dict(obj["trainingRuntimeSpec"]) if obj.get("trainingRuntimeSpec") is not None else None
})
return _obj
api/python_api/kubeflow_trainer_api/models/trainer_v1alpha1_train_job_spec.py
Generated
+10 -14
View File
@@ -20,7 +20,7 @@ import json
from pydantic import BaseModel, ConfigDict, Field, StrictBool, StrictStr
from typing import Any, ClassVar, Dict, List, Optional
from kubeflow_trainer_api.models.trainer_v1alpha1_initializer import TrainerV1alpha1Initializer
from kubeflow_trainer_api.models.trainer_v1alpha1_pod_template_override import TrainerV1alpha1PodTemplateOverride
from kubeflow_trainer_api.models.trainer_v1alpha1_runtime_patch import TrainerV1alpha1RuntimePatch
from kubeflow_trainer_api.models.trainer_v1alpha1_runtime_ref import TrainerV1alpha1RuntimeRef
from kubeflow_trainer_api.models.trainer_v1alpha1_trainer import TrainerV1alpha1Trainer
from typing import Optional, Set
@@ -30,15 +30,13 @@ class TrainerV1alpha1TrainJobSpec(BaseModel):
"""
TrainJobSpec represents specification of the desired TrainJob.
""" # noqa: E501
annotations: Optional[Dict[str, StrictStr]] = Field(default=None, description="annotations to apply for the derivative JobSet and Jobs. They will be merged with the TrainingRuntime values.")
initializer: Optional[TrainerV1alpha1Initializer] = Field(default=None, description="initializer defines the configuration of the initializer.")
labels: Optional[Dict[str, StrictStr]] = Field(default=None, description="labels to apply for the derivative JobSet and Jobs. They will be merged with the TrainingRuntime values.")
managed_by: Optional[StrictStr] = Field(default=None, description="managedBy is used to indicate the controller or entity that manages a TrainJob. The value must be either an empty, `trainer.kubeflow.org/trainjob-controller` or `kueue.x-k8s.io/multikueue`. The built-in TrainJob controller reconciles TrainJob which don't have this field at all or the field value is the reserved string `trainer.kubeflow.org/trainjob-controller`, but delegates reconciling TrainJobs with a 'kueue.x-k8s.io/multikueue' to the Kueue. The field is immutable.", alias="managedBy")
pod_template_overrides: Optional[List[TrainerV1alpha1PodTemplateOverride]] = Field(default=None, description="podTemplateOverrides define the PodTemplateOverrides for the training runtime. When multiple overrides apply to the same targetJob, later entries in the slice override earlier field values.", alias="podTemplateOverrides")
runtime_patches: Optional[List[TrainerV1alpha1RuntimePatch]] = Field(default=None, description="runtimePatches defines custom patches applied to the TrainJob's Runtime. Patches are keyed by manager to provide clear ownership and avoid conflicts between controllers.", alias="runtimePatches")
runtime_ref: TrainerV1alpha1RuntimeRef = Field(description="runtimeRef is the reference to the training runtime.", alias="runtimeRef")
suspend: Optional[StrictBool] = Field(default=None, description="suspend defines whether to suspend the running TrainJob.")
trainer: Optional[TrainerV1alpha1Trainer] = Field(default=None, description="trainer defines the configuration of the trainer.")
__properties: ClassVar[List[str]] = ["annotations", "initializer", "labels", "managedBy", "podTemplateOverrides", "runtimeRef", "suspend", "trainer"]
__properties: ClassVar[List[str]] = ["initializer", "managedBy", "runtimePatches", "runtimeRef", "suspend", "trainer"]
model_config = ConfigDict(
populate_by_name=True,
@@ -82,13 +80,13 @@ class TrainerV1alpha1TrainJobSpec(BaseModel):
# override the default output from pydantic by calling `to_dict()` of initializer
if self.initializer:
_dict['initializer'] = self.initializer.to_dict()
# override the default output from pydantic by calling `to_dict()` of each item in pod_template_overrides (list)
# override the default output from pydantic by calling `to_dict()` of each item in runtime_patches (list)
_items = []
if self.pod_template_overrides:
for _item_pod_template_overrides in self.pod_template_overrides:
if _item_pod_template_overrides:
_items.append(_item_pod_template_overrides.to_dict())
_dict['podTemplateOverrides'] = _items
if self.runtime_patches:
for _item_runtime_patches in self.runtime_patches:
if _item_runtime_patches:
_items.append(_item_runtime_patches.to_dict())
_dict['runtimePatches'] = _items
# override the default output from pydantic by calling `to_dict()` of runtime_ref
if self.runtime_ref:
_dict['runtimeRef'] = self.runtime_ref.to_dict()
@@ -107,11 +105,9 @@ class TrainerV1alpha1TrainJobSpec(BaseModel):
return cls.model_validate(obj)
_obj = cls.model_validate({
"annotations": obj.get("annotations"),
"initializer": TrainerV1alpha1Initializer.from_dict(obj["initializer"]) if obj.get("initializer") is not None else None,
"labels": obj.get("labels"),
"managedBy": obj.get("managedBy"),
"podTemplateOverrides": [TrainerV1alpha1PodTemplateOverride.from_dict(_item) for _item in obj["podTemplateOverrides"]] if obj.get("podTemplateOverrides") is not None else None,
"runtimePatches": [TrainerV1alpha1RuntimePatch.from_dict(_item) for _item in obj["runtimePatches"]] if obj.get("runtimePatches") is not None else None,
"runtimeRef": TrainerV1alpha1RuntimeRef.from_dict(obj["runtimeRef"]) if obj.get("runtimeRef") is not None else None,
"suspend": obj.get("suspend"),
"trainer": TrainerV1alpha1Trainer.from_dict(obj["trainer"]) if obj.get("trainer") is not None else None
api/python_api/kubeflow_trainer_api/models/trainer_v1alpha1_training_runtime_spec_patch.py
Generated
+91
View File
@@ -0,0 +1,91 @@
# coding: utf-8
"""
Kubeflow Trainer OpenAPI Spec
No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator)
The version of the OpenAPI document: unversioned
Generated by OpenAPI Generator (https://openapi-generator.tech)
Do not edit the class manually.
""" # noqa: E501
from __future__ import annotations
import pprint
import re # noqa: F401
import json
from pydantic import BaseModel, ConfigDict, Field
from typing import Any, ClassVar, Dict, List, Optional
from kubeflow_trainer_api.models.trainer_v1alpha1_job_set_template_patch import TrainerV1alpha1JobSetTemplatePatch
from typing import Optional, Set
from typing_extensions import Self
class TrainerV1alpha1TrainingRuntimeSpecPatch(BaseModel):
"""
TrainingRuntimeSpecPatch mirrors TrainingRuntimeSpec but only exposes the fields managers are permitted to patch.
""" # noqa: E501
template: Optional[TrainerV1alpha1JobSetTemplatePatch] = Field(default=None, description="template patches the JobSet template.")
__properties: ClassVar[List[str]] = ["template"]
model_config = ConfigDict(
populate_by_name=True,
validate_assignment=True,
protected_namespaces=(),
)
def to_str(self) -> str:
"""Returns the string representation of the model using alias"""
return pprint.pformat(self.model_dump(by_alias=True))
def to_json(self) -> str:
"""Returns the JSON representation of the model using alias"""
# TODO: pydantic v2: use .model_dump_json(by_alias=True, exclude_unset=True) instead
return json.dumps(self.to_dict())
@classmethod
def from_json(cls, json_str: str) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1TrainingRuntimeSpecPatch from a JSON string"""
return cls.from_dict(json.loads(json_str))
def to_dict(self) -> Dict[str, Any]:
"""Return the dictionary representation of the model using alias.
This has the following differences from calling pydantic's
`self.model_dump(by_alias=True)`:
* `None` is only added to the output dict for nullable fields that
were set at model initialization. Other fields with value `None`
are ignored.
"""
excluded_fields: Set[str] = set([
])
_dict = self.model_dump(
by_alias=True,
exclude=excluded_fields,
exclude_none=True,
)
# override the default output from pydantic by calling `to_dict()` of template
if self.template:
_dict['template'] = self.template.to_dict()
return _dict
@classmethod
def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]:
"""Create an instance of TrainerV1alpha1TrainingRuntimeSpecPatch from a dict"""
if obj is None:
return None
if not isinstance(obj, dict):
return cls.model_validate(obj)
_obj = cls.model_validate({
"template": TrainerV1alpha1JobSetTemplatePatch.from_dict(obj["template"]) if obj.get("template") is not None else None
})
return _obj
charts/kubeflow-trainer/crds/trainer.kubeflow.org_clustertrainingruntimes.yaml
+1
View File
@@ -94,6 +94,7 @@ spec:
description: |-
sshAuthMountPath is the directory where SSH keys are mounted.
Defaults to /root/.ssh.
maxLength: 4096
type: string
type: object
numNodes:
charts/kubeflow-trainer/crds/trainer.kubeflow.org_trainingruntimes.yaml
+1
View File
@@ -94,6 +94,7 @@ spec:
description: |-
sshAuthMountPath is the directory where SSH keys are mounted.
Defaults to /root/.ssh.
maxLength: 4096
type: string
type: object
numNodes:
charts/kubeflow-trainer/crds/trainer.kubeflow.org_trainjobs.yaml
+5048 -4143
View File
File diff suppressed because it is too large Load Diff
docs/proposals/2170-kubeflow-trainer-v2/README.md
+294 -115
View File
@@ -74,6 +74,10 @@ Based on the above personas, we should build an API that everyone will benefit f
For example, training using MPI orchestration.
- Introduce Kubeflow `TrainJob` API that allows to reuse these runtimes and quickly start a new
training job without understanding complex Kubernetes APIs.
- Enable `TrainJob` to extend `TrainingRuntime` defaults through the `RuntimePatches` API — a typed,
multi-owner patch mechanism that lets users, external controllers (e.g. Kueue), and admission
webhooks each contribute a named patch entry to configure JobSet/Job metadata and Pod specs
(e.g. volumes, scheduling directives, etc.) without conflicting with one another.
- Update Kubeflow Training SDK to allow data scientists quickly create and monitor `TrainJobs`.
- Create community-supported `ClusterTrainingRuntime` for distributed training with PyTorch and MPI.
- Create community-supported `ClusterTrainingRuntime` for LLM fine-tuning for various foundational
@@ -312,17 +316,11 @@ type TrainJobSpec struct {
// Configuration of the trainer.
Trainer *Trainer `json:"trainer,omitempty"`
// Labels to apply for the derivative JobSet and Jobs.
// They will be merged with the TrainingRuntime values.
Labels map[string]string `json:"labels,omitempty"`
// Annotations to apply for the derivative JobSet and Jobs.
// They will be merged with the TrainingRuntime values.
Annotations map[string]string `json:"annotations,omitempty"`
// Custom overrides for the training runtime.
// +listType=atomic
PodTemplateOverrides []PodTemplateOverride `json:"podTemplateOverrides,omitempty"`
// runtimePatches defines custom patches applied to the TrainJob's Runtime.
// Patches are keyed by manager to provide clear ownership and avoid conflicts between controllers.
// +listType=map
// +listMapKey=manager
RuntimePatches []RuntimePatch `json:"runtimePatches,omitempty"`
// Whether the controller should suspend the running TrainJob.
// Defaults to false.
@@ -427,18 +425,12 @@ This table explains the rationale for each `TrainJob` parameter:
</td>
</tr>
<tr>
<td><code>Labels and Annotations</code>
<td><code>RuntimePatches</code>
</td>
<td>Custom metadata that needs to be applied to the <code>TrainJob</code> resources: JobSet, Job, and Pods.
</td>
</tr>
<tr>
<td><code>PodTemplateOverrides</code>
</td>
<td>Custom overrides that are specific to the <code>TrainJob</code> and need to be applied to the
<code>TrainJob</code> resources. For example, the user identity. Usually, it is managed by
custom admission webhooks that inject data to the <code>TrainJob</code> after the user creates it
via the Python SDK or <code>kubectl</code>
<td>Structured patches applied to the TrainJob's Runtime, keyed by manager.
Used to inject runtime-specific configuration such as user identity, PVC mounts, node selectors,
or scheduling constraints. Typically managed by admission webhooks or external controllers
(e.g. Kueue) after the user creates the <code>TrainJob</code> via the Python SDK or <code>kubectl</code>.
</td>
</tr>
<tr>
@@ -760,94 +752,228 @@ replicatedJobs:
value: AutoModelForCausalLM
```
### The PodTemplateOverride APIs
### The RuntimePatches API
The `PodTemplateOverride` represents overrides for the `TrainingRuntime` when `TrainJob` is created.
These parameters can include the user's identity or PVC.
`runtimePatches` allows controllers, admission webhooks, and custom clients to attach structured
patches to a `TrainJob` without conflicting with each other. Common use cases include injecting
user identity, PVC mounts, node selectors, or scheduling constraints that are determined
outside the TrainJob itself – for example by Kueue during resource allocation or by a
multi-tenant admission webhook. The patch is applied to the underlying runtime, allowing users
to customize Job template before creation.
Usually, these parameters should not be configured by the user and should be attached during the
orchestration (e.g. using Kubernetes admission webhooks or custom clients).
Each entry is keyed by a unique `manager` field (the list map key). A manager owns its entry
and updates it in place. The `time` field is set by the Trainer admission webhook on each write
and is used for observability only — it is not a list map key.
In the future, we can add more parameters if we find use-cases when it is required.
The `trainingRuntimeSpec` field is a discriminated union over supported runtime kinds. For
`ClusterTrainingRuntime` and `TrainingRuntime`-based jobs it exposes a restricted subset of the
runtime spec: JobSet-level metadata, per-replicated-job Job template metadata, and a curated set
of Pod spec fields.
In the future, we can extend `RuntimePatch` API with other supported Runtimes (e.g. GroveRuntime).
In that case, users will be able to select one of supported specs, for example: `TrainingRuntimeSpec`
or `GroveRuntimeSpec`
![runtime-patches](./runtime-patches.drawio.svg)
#### API Design
```golang
// PodTemplateOverride represents the custom overrides that will be applied for the TrainJob's resources.
type PodTemplateOverride struct {
// TargetJobs are the training job replicas in the training runtime template to apply the overrides.
// +listType=atomic
TargetJobs []PodTemplateOverrideTargetJob `json:"targetJobs"`
// RuntimePatch represents a custom patch applied to the TrainJob's training runtime template.
// Patches are keyed by manager to provide clear ownership and avoid conflicts between controllers.
type RuntimePatch struct {
// manager indicates who owns this patch entry. It can be set by the user, external
// controllers, or admission webhooks to track ownership and avoid conflicts.
// For example, Kueue sets this field to "kueue.x-k8s.io/manager".
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable"
// +required
Manager string `json:"manager"`
// Override for the Pod template metadata.
// These values will be merged with the TrainingRuntime's Pod template metadata.
// time is the timestamp of when this patch was last written.
// Set by the Trainer admission webhook on each create or update. Used for observability only,
// not used as a list map key.
// +optional
Time *metav1.Time `json:"time,omitempty"`
// trainingRuntimeSpec defines allowed patches for ClusterTrainingRuntime or TrainingRuntime-based jobs.
// +optional
TrainingRuntimeSpec *TrainingRuntimeSpecPatch `json:"trainingRuntimeSpec,omitempty"`
}
// TrainingRuntimeSpecPatch mirrors TrainingRuntimeSpec but only exposes
// the fields managers are permitted to patch.
type TrainingRuntimeSpecPatch struct {
// template patches the JobSet template.
// +optional
Template *JobSetTemplatePatch `json:"template,omitempty"`
}
// JobSetTemplatePatch defines patches for the JobSet template.
// It mirrors JobSetTemplateSpec but only exposes metadata and restricted spec fields.
type JobSetTemplatePatch struct {
// metadata patches the JobSet object metadata.
// Only labels and annotations are allowed.
// +optional
Metadata *metav1.ObjectMeta `json:"metadata,omitempty"`
// Override for the Pod template spec.
// These values will be merged with the TrainingRuntime's Pod template spec.
Spec *PodTemplateSpecOverride `json:"spec,omitempty"`
// spec patches the JobSet spec with restricted fields.
// +optional
Spec *JobSetSpecPatch `json:"spec,omitempty"`
}
type PodTemplateOverrideTargetJob struct {
// Name is the target training job name for which the PodSpec is overridden.
// JobSetSpecPatch defines allowed patches for the JobSet spec.
type JobSetSpecPatch struct {
// replicatedJobs defines per-job patches, keyed by job name.
// +listType=map
// +listMapKey=name
// +optional
ReplicatedJobs []ReplicatedJobPatch `json:"replicatedJobs,omitempty"`
}
// ReplicatedJobPatch defines patches for a specific replicated job within the JobSet.
type ReplicatedJobPatch struct {
// name is the name of the replicated job to patch.
// +kubebuilder:validation:MinLength=1
// +required
Name string `json:"name"`
// template patches the Job template for this replicated job.
// +optional
Template *JobTemplatePatch `json:"template,omitempty"`
}
// PodTemplateSpecOverride represents the spec overrides for Pod template.
type PodTemplateSpecOverride struct {
// Override for the service account.
// JobTemplatePatch defines patches for a Job template within a replicated job.
type JobTemplatePatch struct {
// metadata patches the Job template metadata.
// Only labels and annotations are allowed.
// +optional
Metadata *metav1.ObjectMeta `json:"metadata,omitempty"`
// spec patches the Job spec with restricted fields.
// +optional
Spec *JobSpecPatch `json:"spec,omitempty"`
}
// JobSpecPatch defines allowed patches for the Job spec.
type JobSpecPatch struct {
// template patches the Pod template for this Job.
// +optional
Template *PodTemplatePatch `json:"template,omitempty"`
}
// PodTemplatePatch defines patches for a Pod template within a Job.
type PodTemplatePatch struct {
// metadata patches the Pod template metadata.
// Only labels and annotations are allowed.
// +optional
Metadata *metav1.ObjectMeta `json:"metadata,omitempty"`
// spec patches the Pod spec with the fields managers are permitted to set.
// +optional
Spec *PodSpecPatch `json:"spec,omitempty"`
}
// PodSpecPatch contains the Pod spec fields that managers are permitted to patch.
type PodSpecPatch struct {
// serviceAccountName patches the service account for the Pods in the target job templates.
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable"
// +optional
ServiceAccountName *string `json:"serviceAccountName,omitempty"`
// Override for the node selector to place Pod on the specific mode.
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
// Override for the Pod's affinity.
Affinity *corev1.Affinity `json:"affinity,omitempty"`
// Override for the Pod's tolerations.
Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
// Overrides for the Pod volume configuration.
// volumes patches the Pod's volumes.
// +listType=map
// +listMapKey=name
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable"
// +optional
Volumes []corev1.Volume `json:"volumes,omitempty"`
// Overrides for the init container in the desired job templates.
InitContainers []ContainerOverride `json:"initContainers,omitempty"`
// initContainers patches the init containers in the target job templates.
// +listType=map
// +listMapKey=name
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable"
// +optional
InitContainers []ContainerPatch `json:"initContainers,omitempty"`
// Overrides for the containers in the desired job templates.
Containers []ContainerOverride `json:"containers,omitempty"`
// containers patches specific containers in the target job templates.
// +listType=map
// +listMapKey=name
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable"
// +optional
Containers []ContainerPatch `json:"containers,omitempty"`
// SchedulingGates overrides the scheduling gates of the Pods in the target job templates.
SchedulingGates []corev1.PodSchedulingGate `json:"schedulingGates,omitempty"`
// ImagePullSecrets overrides the image pull secrets for the Pods in the target job templates.
// imagePullSecrets patches the image pull secrets for the Pods in the target job templates.
// +listType=map
// +listMapKey=name
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable"
// +optional
ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
// securityContext patches the Pod's security context.
// More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable"
// +optional
SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"`
// nodeSelector patches the node selector to place Pods on specific nodes.
// +optional
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
// affinity patches the Pod's scheduling affinity.
// +optional
Affinity *corev1.Affinity `json:"affinity,omitempty"`
// tolerations patches the Pod's tolerations.
// +listType=atomic
// +optional
Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
// schedulingGates patches the scheduling gates for the Pods in the target job templates.
// More info: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-scheduling-readiness/
// +listType=map
// +listMapKey=name
// +optional
SchedulingGates []corev1.PodSchedulingGate `json:"schedulingGates,omitempty"`
}
// ContainerOverride represents parameters that can be overridden using PodTemplateOverrides.
type ContainerOverride struct {
// Name for the container. TrainingRuntime must have this container.
Name string `json:"name"`
// ContainerPatch represents parameters that can be patched using PodSpecPatch.
type ContainerPatch struct {
// name for the container. Runtime must have this container.
// +kubebuilder:validation:MinLength=1
// +required
Name string `json:"name,omitempty"`
// List of environment variables to set in the container.
// These values will be merged with the TrainingRuntime's environments.
// This value can't be set for container with the name: `node`, `dataset-initializer`, or
// `model-initializer`. For those containers the envs can be set via Trainer or Initializer APIs.
// env is the list of environment variables to set in the container.
// These values will be merged with the Runtime's environments.
// These values can't be set for container with the name: `node`, `dataset-initializer`, or
// `model-initializer`. For those containers the envs can only be set via Trainer or Initializer APIs.
// +listType=map
// +listMapKey=name
// +optional
Env []corev1.EnvVar `json:"env,omitempty"`
// Pod volumes to mount into the container's filesystem.
// volumeMounts are the volumes to mount into the container's filesystem.
// +listType=map
// +listMapKey=name
// +optional
VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"`
// securityContext patches the container's security context.
// More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
// +optional
SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"`
}
```
The webhook will validate that TargetJob and Container name exist in the Runtime Job template.
The webhook validates that the container names in `Containers` and `InitContainers` exist in
the Runtime's Job template. The patches are applied during the build phase of the
[Pipelines Framework](#pipeline-framework) in the `ComponentBuilder` plugin. the `RuntimePatches`
update values in the TrainJob's Runtime template, since they contain the final
desired values for the underlying Job.
The overrides will be applied during the build phase of [Pipelines Framework](#pipeline-framework)
in the `ComponentBuilder` plugin.
#### Example of TrainJob with Patches
The PodTemplateOverrides will override values of TrainJob and Runtime Job template,
since it should contain the final value for the underlying Job.
#### Example of TrainJob with Overrides
This example shows how to override the user-identity for the sidecar container and add volume to the
This example shows how to patch the user-identity for the sidecar container and add volume to the
trainer container.
```yaml
@@ -861,27 +987,34 @@ spec:
name: pytorch-distributed-gpu
trainer:
image: docker.io/custom-training
podTemplateOverrides:
- targetJobs:
- name: node
spec:
initContainers:
- name: fetch-identity
env:
- name: USER_ID
value: "123"
containers:
- name: trainer
volumeMounts:
- name: user-123-volume
mountPath: /workspace
volumes:
- name: user-123-volume
persistentVolumeClaim:
claimName: user-123-volume
runtimePatches:
- manager: trainer.kubeflow.org/kubeflow-sdk
time: "2026-05-01T15:20:00Z"
trainingRuntimeSpec:
template:
spec:
replicatedJobs:
- name: node
template:
metadata:
labels:
custom-label: value
spec:
template:
spec:
containers:
- name: trainer
volumeMounts:
- name: user-123-volume
mountPath: /workspace
volumes:
- name: user-123-volume
persistentVolumeClaim:
claimName: user-123-volume
```
Users can also define multiple PodTemplateOverrides for every ReplicatedJob:
The same manager can update its patch entry during the lifecycle of a TrainJob.
For example, Kueue might update node selectors as resources become available:
```yaml
apiVersion: trainer.kubeflow.org/v2alpha1
@@ -894,25 +1027,70 @@ spec:
name: pytorch-distributed-gpu
trainer:
image: docker.io/custom-training
podTemplateOverrides:
- targetJobs:
- name: dataset-initializer
spec:
initContainers:
- name: fetch-identity
env:
- name: USER_ID
value: "123"
- targetJobs:
- name: node
spec:
initContainers:
- name: fetch-identity
env:
- name: USER_ID
value: "123"
runtimePatches:
- manager: kueue.x-k8s.io/manager
time: "2026-02-05T10:10:00Z"
trainingRuntimeSpec:
template:
spec:
replicatedJobs:
- name: node
template:
spec:
template:
spec:
nodeSelector:
node-type: gpu-a100
zone: us-west-1b
```
#### Support Arbitrary Patches (Future Extension)
The long-term goal of the Trainer Extension Framework is to support arbitrary runtimes, including
operators that are not part of the upstream Trainer project, such as in-house or third-party runtime
implementations. For example, users may operate a custom operator to orchestrate distributed AI
workloads that Trainer has no schema knowledge of.
To enable this flexibility, the `runtimePatches` API supports custom runtimes through an opaque spec
patch field. One approach that balances flexibility with API simplicity is to store the spec patch as
a raw object:
```go
// +kubebuilder:validation:XValidation:rule="[has(self.trainingRuntimeSpec), has(self.opaqueRuntimeSpec)].filter(x, x).size() <= 1", message="only one spec can be set"
type RuntimePatch struct {
Manager string `json:"manager"`
Time *metav1.Time `json:"time,omitempty"`
TrainingRuntimeSpec *TrainingRuntimeSpecPatch `json:"trainingRuntimeSpec,omitempty"`
// opaqueRuntimeSpec stores the full spec patch for runtimes whose schema is unknown to Trainer.
// Used when runtimeRef points to a runtime kind not natively supported by Trainer.
// +optional
OpaqueRuntimeSpec *runtime.RawExtension `json:"opaqueRuntimeSpec,omitempty"`
}
```
The following example shows how `opaqueRuntimeSpec` can represent a patch for a `SparkApplication` CRD:
```yaml
runtimePatches:
- manager: kueue.x-k8s.io/manager
time: "2026-02-05T10:10:00Z"
opaqueRuntimeSpec:
driver: # Spark Driver spec
template: # Valid PodTemplateSpec
spec:
nodeSelector:
node-type: gpu-a100
executor: # Spark Executor spec
template: # Valid PodTemplateSpec
spec:
nodeSelector:
node-type: gpu-a100
```
Since Trainer has no schema knowledge of the target runtime, `opaqueRuntimeSpec` content is not
validated at admission time beyond confirming it is a valid JSON object. Validation occurs at
reconcile time when the patch is applied to the target CR.
### State Transition
In this section, we're explaining the TrainJob state transition (`.status.conditions`).
@@ -1763,6 +1941,7 @@ spec:
- 2026-02-23 Remove ElasticPolicy from the Torch API – will be implemented in the future KEPs
- 2026-02-23 Removed `numProcPerNode` from `TorchMLPolicySource`; `TrainJob.Trainer.numProcPerNode`
now accepts only int values (torch plugin defaults to `auto`)
- 2025-10-09 Replace PodTemplateOverrides with RuntimePatches API
## Alternatives
docs/proposals/2170-kubeflow-trainer-v2/runtime-patches.drawio.svg
+4
View File
File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 1.1 MiB

manifests/base/crds/trainer.kubeflow.org_clustertrainingruntimes.yaml
+1
View File
@@ -94,6 +94,7 @@ spec:
description: |-
sshAuthMountPath is the directory where SSH keys are mounted.
Defaults to /root/.ssh.
maxLength: 4096
type: string
type: object
numNodes:
manifests/base/crds/trainer.kubeflow.org_trainingruntimes.yaml
+1
View File
@@ -94,6 +94,7 @@ spec:
description: |-
sshAuthMountPath is the directory where SSH keys are mounted.
Defaults to /root/.ssh.
maxLength: 4096
type: string
type: object
numNodes:
manifests/base/crds/trainer.kubeflow.org_trainjobs.yaml
+5048 -4143
View File
File diff suppressed because it is too large Load Diff
pkg/apis/config/v1alpha1/configuration_types.go
+7
View File
@@ -79,6 +79,7 @@ type ControllerWebhook struct {
// host is the hostname that the webhook server binds to.
// It is used to set webhook.Server.Host.
// Defaults to "" (all interfaces).
// +kubebuilder:validation:MaxLength=253
// +optional
Host *string `json:"host,omitempty"`
}
@@ -93,6 +94,7 @@ type ControllerMetrics struct {
// +optional
// +kubebuilder:default=":8443"
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=255
BindAddress string `json:"bindAddress,omitempty"`
// secureServing determines if the metrics endpoint should be served securely via HTTPS.
@@ -112,6 +114,7 @@ type ControllerHealth struct {
// +optional
// +kubebuilder:default=":8081"
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=255
HealthProbeBindAddress string `json:"healthProbeBindAddress,omitempty"`
// readinessEndpointName is the name for the readiness endpoint.
@@ -119,6 +122,7 @@ type ControllerHealth struct {
// +optional
// +kubebuilder:default="readyz"
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=63
ReadinessEndpointName string `json:"readinessEndpointName,omitempty"`
// livenessEndpointName is the name for the liveness endpoint.
@@ -126,6 +130,7 @@ type ControllerHealth struct {
// +optional
// +kubebuilder:default="healthz"
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=63
LivenessEndpointName string `json:"livenessEndpointName,omitempty"`
}
@@ -163,6 +168,7 @@ type CertManagement struct {
// +optional
// +kubebuilder:default="kubeflow-trainer-controller-manager"
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=253
WebhookServiceName string `json:"webhookServiceName,omitempty"`
// webhookSecretName is the name of the Secret used to store the CA and server certificates.
@@ -170,6 +176,7 @@ type CertManagement struct {
// +optional
// +kubebuilder:default="kubeflow-trainer-webhook-cert"
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=253
WebhookSecretName string `json:"webhookSecretName,omitempty"`
}
pkg/apis/trainer/v1alpha1/trainingruntime_types.go
+1
View File
@@ -239,6 +239,7 @@ type MPIMLPolicySource struct {
// sshAuthMountPath is the directory where SSH keys are mounted.
// Defaults to /root/.ssh.
// +kubebuilder:default=/root/.ssh
// +kubebuilder:validation:MaxLength=4096
// +optional
SSHAuthMountPath *string `json:"sshAuthMountPath,omitempty"`
pkg/apis/trainer/v1alpha1/trainjob_types.go
+160 -71
View File
@@ -110,21 +110,13 @@ type TrainJobSpec struct {
// +optional
Trainer *Trainer `json:"trainer,omitempty"`
// labels to apply for the derivative JobSet and Jobs.
// They will be merged with the TrainingRuntime values.
// runtimePatches defines custom patches applied to the TrainJob's Runtime.
// Patches are keyed by manager to provide clear ownership and avoid conflicts between controllers.
// +listType=map
// +listMapKey=manager
// +kubebuilder:validation:MaxItems=16
// +optional
Labels map[string]string `json:"labels,omitempty"`
// annotations to apply for the derivative JobSet and Jobs.
// They will be merged with the TrainingRuntime values.
// +optional
Annotations map[string]string `json:"annotations,omitempty"`
// podTemplateOverrides define the PodTemplateOverrides for the training runtime.
// When multiple overrides apply to the same targetJob, later entries in the slice override earlier field values.
// +listType=atomic
// +optional
PodTemplateOverrides []PodTemplateOverride `json:"podTemplateOverrides,omitempty"`
RuntimePatches []RuntimePatch `json:"runtimePatches,omitempty"`
// suspend defines whether to suspend the running TrainJob.
// +kubebuilder:default=false
@@ -139,6 +131,7 @@ type TrainJobSpec struct {
// with a 'kueue.x-k8s.io/multikueue' to the Kueue. The field is immutable.
// +kubebuilder:default="trainer.kubeflow.org/trainjob-controller"
// +kubebuilder:validation:XValidation:rule="self in ['trainer.kubeflow.org/trainjob-controller', 'kueue.x-k8s.io/multikueue']", message="ManagedBy must be trainer.kubeflow.org/trainjob-controller or kueue.x-k8s.io/multikueue if set"
// +kubebuilder:validation:MaxLength=253
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable"
// +optional
ManagedBy *string `json:"managedBy,omitempty"`
@@ -150,18 +143,21 @@ type RuntimeRef struct {
// When namespaced-scoped TrainingRuntime is used, the TrainJob must have
// the same namespace as the deployed runtime.
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=253
// +required
Name string `json:"name,omitempty"`
// apiGroup of the runtime being referenced.
// Defaults to `trainer.kubeflow.org`.
// +kubebuilder:default="trainer.kubeflow.org"
// +kubebuilder:validation:MaxLength=253
// +optional
APIGroup *string `json:"apiGroup,omitempty"`
// kind of the runtime being referenced.
// Defaults to ClusterTrainingRuntime.
// +kubebuilder:default="ClusterTrainingRuntime"
// +kubebuilder:validation:MaxLength=253
// +optional
Kind *string `json:"kind,omitempty"`
}
@@ -183,6 +179,7 @@ type Initializer struct {
// which contains this label: `trainer.kubeflow.org/trainjob-ancestor-step: dataset-initializer`
type DatasetInitializer struct {
// storageUri is the URI for the dataset provider.
// +kubebuilder:validation:MaxLength=2048
// +optional
StorageUri *string `json:"storageUri,omitempty"`
@@ -190,6 +187,7 @@ type DatasetInitializer struct {
// These values will be merged with the TrainingRuntime's dataset initializer environments.
// +listType=map
// +listMapKey=name
// +kubebuilder:validation:MaxItems=128
// +optional
Env []corev1.EnvVar `json:"env,omitempty"`
@@ -204,6 +202,7 @@ type DatasetInitializer struct {
// which contains this label: `trainer.kubeflow.org/trainjob-ancestor-step: dataset-initializer`
type ModelInitializer struct {
// storageUri is the URI for the model provider.
// +kubebuilder:validation:MaxLength=2048
// +optional
StorageUri *string `json:"storageUri,omitempty"`
@@ -211,6 +210,7 @@ type ModelInitializer struct {
// These values will be merged with the TrainingRuntime's model initializer environments.
// +listType=map
// +listMapKey=name
// +kubebuilder:validation:MaxItems=128
// +optional
Env []corev1.EnvVar `json:"env,omitempty"`
@@ -225,16 +225,21 @@ type ModelInitializer struct {
// which contains this label: `trainer.kubeflow.org/trainjob-ancestor-step: trainer`
type Trainer struct {
// image is the container image for the training container.
// +kubebuilder:validation:MaxLength=500
// +optional
Image *string `json:"image,omitempty"`
// command for the entrypoint of the training container.
// +listType=atomic
// +kubebuilder:validation:MaxItems=128
// +kubebuilder:validation:items:MaxLength=65536
// +optional
Command []string `json:"command,omitempty"`
// args for the entrypoint for the training container.
// +listType=atomic
// +kubebuilder:validation:MaxItems=128
// +kubebuilder:validation:items:MaxLength=65536
// +optional
Args []string `json:"args,omitempty"`
@@ -242,6 +247,7 @@ type Trainer struct {
// These values will be merged with the TrainingRuntime's trainer environments.
// +listType=map
// +listMapKey=name
// +kubebuilder:validation:MaxItems=128
// +optional
Env []corev1.EnvVar `json:"env,omitempty"`
@@ -261,119 +267,199 @@ type Trainer struct {
NumProcPerNode *int32 `json:"numProcPerNode,omitempty"`
}
// PodTemplateOverride represents a custom PodTemplateSpec override that will be applied to the TrainJob's training runtime.
type PodTemplateOverride struct {
// targetJobs is the list of replicated jobs in the training runtime template to apply the overrides.
// +listType=atomic
// RuntimePatch represents a custom patch applied to the TrainJob's training runtime template.
// Patches are keyed by manager to provide clear ownership and avoid conflicts between controllers.
type RuntimePatch struct {
// manager indicates who owns this patch entry. It can be set by the user, external
// controllers, or admission webhooks to track ownership and avoid conflicts.
// For example, Kueue sets this field to "kueue.x-k8s.io/manager".
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=253
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable"
// +required
TargetJobs []PodTemplateOverrideTargetJob `json:"targetJobs,omitempty"`
Manager string `json:"manager,omitempty"`
// manager indicates which controller created this PodTemplateOverride object.
// It can be used by external controllers or admission webhooks to track ownership
// and avoid conflicts with other controllers. For example, Kueue sets this field
// to "kueue.k8s.io/manager".
// Defaults to `trainer.kubeflow.org/unknown`
// +kubebuilder:default="trainer.kubeflow.org/unknown"
// time is the timestamp of when this patch was last written.
// Set by the Trainer admission webhook on each create or update. Used for observability only,
// not used as a list map key.
// +optional
Manager *string `json:"manager,omitempty"`
Time *metav1.Time `json:"time,omitempty"`
// metadata overrides the Pod template metadata.
// These values will be merged with the TrainingRuntime's Pod template metadata.
// trainingRuntimeSpec defines allowed patches for ClusterTrainingRuntime or TrainingRuntime-based jobs.
// +optional
TrainingRuntimeSpec *TrainingRuntimeSpecPatch `json:"trainingRuntimeSpec,omitempty"`
}
// TrainingRuntimeSpecPatch mirrors TrainingRuntimeSpec but only exposes
// the fields managers are permitted to patch.
type TrainingRuntimeSpecPatch struct {
// template patches the JobSet template.
// +optional
Template *JobSetTemplatePatch `json:"template,omitempty"`
}
// JobSetTemplatePatch defines patches for the JobSet template.
// It mirrors JobSetTemplateSpec but only exposes metadata and restricted spec fields.
type JobSetTemplatePatch struct {
// metadata patches the JobSet object metadata.
// Only labels and annotations are allowed.
// +optional
Metadata *metav1.ObjectMeta `json:"metadata,omitempty"`
// spec overrides the Pod template spec.
// These values will be merged with the TrainingRuntime's Pod template spec.
// spec patches the JobSet spec with restricted fields.
// +optional
Spec *PodTemplateSpecOverride `json:"spec,omitempty"`
Spec *JobSetSpecPatch `json:"spec,omitempty"`
}
type PodTemplateOverrideTargetJob struct {
// name is the target training job name for which the PodTemplateSpec is overridden.
// JobSetSpecPatch defines allowed patches for the JobSet spec.
type JobSetSpecPatch struct {
// replicatedJobs defines per-job patches, keyed by job name.
// +listType=map
// +listMapKey=name
// +kubebuilder:validation:MaxItems=100
// +optional
ReplicatedJobs []ReplicatedJobPatch `json:"replicatedJobs,omitempty"`
}
// ReplicatedJobPatch defines patches for a specific replicated job within the JobSet.
type ReplicatedJobPatch struct {
// name is the name of the replicated job to patch.
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=253
// +required
Name string `json:"name,omitempty"`
// template patches the Job template for this replicated job.
// +optional
Template *JobTemplatePatch `json:"template,omitempty"`
}
// PodTemplateSpecOverride represents the spec overrides for Pod template.
type PodTemplateSpecOverride struct {
// serviceAccountName overrides the service account.
// JobTemplatePatch defines patches for a Job template within a replicated job.
type JobTemplatePatch struct {
// metadata patches the Job template metadata.
// Only labels and annotations are allowed.
// +optional
Metadata *metav1.ObjectMeta `json:"metadata,omitempty"`
// spec patches the Job spec with restricted fields.
// +optional
Spec *JobSpecPatch `json:"spec,omitempty"`
}
// JobSpecPatch defines allowed patches for the Job spec.
type JobSpecPatch struct {
// template patches the Pod template for this Job.
// +optional
Template *PodTemplatePatch `json:"template,omitempty"`
}
// PodTemplatePatch defines patches for a Pod template within a Job.
type PodTemplatePatch struct {
// metadata patches the Pod template metadata.
// Only labels and annotations are allowed.
// +optional
Metadata *metav1.ObjectMeta `json:"metadata,omitempty"`
// spec patches the Pod spec with the fields managers are permitted to set.
// +optional
Spec *PodSpecPatch `json:"spec,omitempty"`
}
// PodSpecPatch contains the Pod spec fields that managers are permitted to patch.
type PodSpecPatch struct {
// serviceAccountName patches the service account for the Pods in the target job templates.
// +kubebuilder:validation:MaxLength=253
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable"
// +optional
ServiceAccountName *string `json:"serviceAccountName,omitempty"`
// nodeSelector overrides the node selector to place Pod on the specific node.
// +optional
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
// affinity overrides for the Pod's affinity.
// +optional
Affinity *corev1.Affinity `json:"affinity,omitempty"`
// tolerations overrides the Pod's tolerations.
// +listType=atomic
// +optional
Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
// securityContext overrides the Pod's security context.
// More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
// +optional
SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"`
// volumes overrides the Pod's volumes.
// volumes patches the Pod's volumes.
// +listType=map
// +listMapKey=name
// +kubebuilder:validation:MaxItems=128
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable"
// +optional
Volumes []corev1.Volume `json:"volumes,omitempty"`
// initContainers overrides the init container in the target job templates.
// initContainers patches the init containers in the target job templates.
// +listType=map
// +listMapKey=name
// +kubebuilder:validation:MaxItems=64
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable"
// +optional
InitContainers []ContainerOverride `json:"initContainers,omitempty"`
InitContainers []ContainerPatch `json:"initContainers,omitempty"`
// containers overrides for the containers in the target job templates.
// containers patches specific containers in the target job templates.
// +listType=map
// +listMapKey=name
// +kubebuilder:validation:MaxItems=64
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable"
// +optional
Containers []ContainerOverride `json:"containers,omitempty"`
Containers []ContainerPatch `json:"containers,omitempty"`
// schedulingGates overrides the scheduling gates of the Pods in the target job templates.
// imagePullSecrets patches the image pull secrets for the Pods in the target job templates.
// +listType=map
// +listMapKey=name
// +kubebuilder:validation:MaxItems=64
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable"
// +optional
ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
// securityContext patches the Pod's security context.
// More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="field is immutable"
// +optional
SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"`
// nodeSelector patches the node selector to place Pods on specific nodes.
// +optional
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
// affinity patches the Pod's scheduling affinity.
// +optional
Affinity *corev1.Affinity `json:"affinity,omitempty"`
// tolerations patches the Pod's tolerations.
// +listType=atomic
// +kubebuilder:validation:MaxItems=128
// +optional
Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
// schedulingGates patches the scheduling gates for the Pods in the target job templates.
// More info: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-scheduling-readiness/
// +listType=map
// +listMapKey=name
// +kubebuilder:validation:MaxItems=16
// +optional
SchedulingGates []corev1.PodSchedulingGate `json:"schedulingGates,omitempty"`
// imagePullSecrets overrides the image pull secrets for the Pods in the target job templates.
// +listType=map
// +listMapKey=name
// +optional
ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
}
// ContainerOverride represents parameters that can be overridden using PodSpecOverrides.
type ContainerOverride struct {
// name for the container. TrainingRuntime must have this container.
// ContainerPatch represents parameters that can be patched using PodSpecPatch.
type ContainerPatch struct {
// name for the container. Runtime must have this container.
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=253
// +required
Name string `json:"name,omitempty"`
// env is the list of environment variables to set in the container.
// These values will be merged with the TrainingRuntime's environments.
// These values will be merged with the Runtime's environments.
// These values can't be set for container with the name: `node`, `dataset-initializer`, or
// `model-initializer`. For those containers the envs can only be set via Trainer or Initializer APIs.
// +listType=map
// +listMapKey=name
// +kubebuilder:validation:MaxItems=128
// +optional
Env []corev1.EnvVar `json:"env,omitempty"`
// volumeMounts are the volumes to mount into the container's filesystem.
// +listType=map
// +listMapKey=name
// +kubebuilder:validation:MaxItems=128
// +optional
VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"`
// securityContext overrides the container's security context.
// securityContext patches the container's security context.
// More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
// +optional
SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"`
@@ -386,11 +472,13 @@ type TrainJobStatus struct {
// +optional
// +listType=map
// +listMapKey=type
// +kubebuilder:validation:MaxItems=8
Conditions []metav1.Condition `json:"conditions,omitempty"`
// jobsStatus tracks the child Jobs in TrainJob.
// +listType=map
// +listMapKey=name
// +kubebuilder:validation:MaxItems=100
// +optional
JobsStatus []JobStatus `json:"jobsStatus,omitempty"`
}
@@ -398,6 +486,7 @@ type TrainJobStatus struct {
type JobStatus struct {
// name of the child Job.
// +kubebuilder:validation:MinLength=1
// +kubebuilder:validation:MaxLength=253
// +required
Name string `json:"name,omitempty"`
pkg/apis/trainer/v1alpha1/zz_generated.deepcopy.go
Generated
+230 -107
View File
@@ -87,7 +87,7 @@ func (in *ClusterTrainingRuntimeList) DeepCopyObject() runtime.Object {
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ContainerOverride) DeepCopyInto(out *ContainerOverride) {
func (in *ContainerPatch) DeepCopyInto(out *ContainerPatch) {
*out = *in
if in.Env != nil {
in, out := &in.Env, &out.Env
@@ -111,12 +111,12 @@ func (in *ContainerOverride) DeepCopyInto(out *ContainerOverride) {
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContainerOverride.
func (in *ContainerOverride) DeepCopy() *ContainerOverride {
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContainerPatch.
func (in *ContainerPatch) DeepCopy() *ContainerPatch {
if in == nil {
return nil
}
out := new(ContainerOverride)
out := new(ContainerPatch)
in.DeepCopyInto(out)
return out
}
@@ -238,6 +238,55 @@ func (in *JAXMLPolicySource) DeepCopy() *JAXMLPolicySource {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *JobSetSpecPatch) DeepCopyInto(out *JobSetSpecPatch) {
*out = *in
if in.ReplicatedJobs != nil {
in, out := &in.ReplicatedJobs, &out.ReplicatedJobs
*out = make([]ReplicatedJobPatch, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobSetSpecPatch.
func (in *JobSetSpecPatch) DeepCopy() *JobSetSpecPatch {
if in == nil {
return nil
}
out := new(JobSetSpecPatch)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *JobSetTemplatePatch) DeepCopyInto(out *JobSetTemplatePatch) {
*out = *in
if in.Metadata != nil {
in, out := &in.Metadata, &out.Metadata
*out = new(metav1.ObjectMeta)
(*in).DeepCopyInto(*out)
}
if in.Spec != nil {
in, out := &in.Spec, &out.Spec
*out = new(JobSetSpecPatch)
(*in).DeepCopyInto(*out)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobSetTemplatePatch.
func (in *JobSetTemplatePatch) DeepCopy() *JobSetTemplatePatch {
if in == nil {
return nil
}
out := new(JobSetTemplatePatch)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *JobSetTemplateSpec) DeepCopyInto(out *JobSetTemplateSpec) {
*out = *in
@@ -256,6 +305,27 @@ func (in *JobSetTemplateSpec) DeepCopy() *JobSetTemplateSpec {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *JobSpecPatch) DeepCopyInto(out *JobSpecPatch) {
*out = *in
if in.Template != nil {
in, out := &in.Template, &out.Template
*out = new(PodTemplatePatch)
(*in).DeepCopyInto(*out)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobSpecPatch.
func (in *JobSpecPatch) DeepCopy() *JobSpecPatch {
if in == nil {
return nil
}
out := new(JobSpecPatch)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *JobStatus) DeepCopyInto(out *JobStatus) {
*out = *in
@@ -297,6 +367,32 @@ func (in *JobStatus) DeepCopy() *JobStatus {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *JobTemplatePatch) DeepCopyInto(out *JobTemplatePatch) {
*out = *in
if in.Metadata != nil {
in, out := &in.Metadata, &out.Metadata
*out = new(metav1.ObjectMeta)
(*in).DeepCopyInto(*out)
}
if in.Spec != nil {
in, out := &in.Spec, &out.Spec
*out = new(JobSpecPatch)
(*in).DeepCopyInto(*out)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JobTemplatePatch.
func (in *JobTemplatePatch) DeepCopy() *JobTemplatePatch {
if in == nil {
return nil
}
out := new(JobTemplatePatch)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *MLPolicy) DeepCopyInto(out *MLPolicy) {
*out = *in
@@ -473,65 +569,44 @@ func (in *PodGroupPolicySource) DeepCopy() *PodGroupPolicySource {
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *PodTemplateOverride) DeepCopyInto(out *PodTemplateOverride) {
*out = *in
if in.TargetJobs != nil {
in, out := &in.TargetJobs, &out.TargetJobs
*out = make([]PodTemplateOverrideTargetJob, len(*in))
copy(*out, *in)
}
if in.Manager != nil {
in, out := &in.Manager, &out.Manager
*out = new(string)
**out = **in
}
if in.Metadata != nil {
in, out := &in.Metadata, &out.Metadata
*out = new(metav1.ObjectMeta)
(*in).DeepCopyInto(*out)
}
if in.Spec != nil {
in, out := &in.Spec, &out.Spec
*out = new(PodTemplateSpecOverride)
(*in).DeepCopyInto(*out)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodTemplateOverride.
func (in *PodTemplateOverride) DeepCopy() *PodTemplateOverride {
if in == nil {
return nil
}
out := new(PodTemplateOverride)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *PodTemplateOverrideTargetJob) DeepCopyInto(out *PodTemplateOverrideTargetJob) {
*out = *in
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodTemplateOverrideTargetJob.
func (in *PodTemplateOverrideTargetJob) DeepCopy() *PodTemplateOverrideTargetJob {
if in == nil {
return nil
}
out := new(PodTemplateOverrideTargetJob)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *PodTemplateSpecOverride) DeepCopyInto(out *PodTemplateSpecOverride) {
func (in *PodSpecPatch) DeepCopyInto(out *PodSpecPatch) {
*out = *in
if in.ServiceAccountName != nil {
in, out := &in.ServiceAccountName, &out.ServiceAccountName
*out = new(string)
**out = **in
}
if in.Volumes != nil {
in, out := &in.Volumes, &out.Volumes
*out = make([]v1.Volume, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.InitContainers != nil {
in, out := &in.InitContainers, &out.InitContainers
*out = make([]ContainerPatch, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.Containers != nil {
in, out := &in.Containers, &out.Containers
*out = make([]ContainerPatch, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.ImagePullSecrets != nil {
in, out := &in.ImagePullSecrets, &out.ImagePullSecrets
*out = make([]v1.LocalObjectReference, len(*in))
copy(*out, *in)
}
if in.SecurityContext != nil {
in, out := &in.SecurityContext, &out.SecurityContext
*out = new(v1.PodSecurityContext)
(*in).DeepCopyInto(*out)
}
if in.NodeSelector != nil {
in, out := &in.NodeSelector, &out.NodeSelector
*out = make(map[string]string, len(*in))
@@ -551,51 +626,92 @@ func (in *PodTemplateSpecOverride) DeepCopyInto(out *PodTemplateSpecOverride) {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.SecurityContext != nil {
in, out := &in.SecurityContext, &out.SecurityContext
*out = new(v1.PodSecurityContext)
(*in).DeepCopyInto(*out)
}
if in.Volumes != nil {
in, out := &in.Volumes, &out.Volumes
*out = make([]v1.Volume, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.InitContainers != nil {
in, out := &in.InitContainers, &out.InitContainers
*out = make([]ContainerOverride, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.Containers != nil {
in, out := &in.Containers, &out.Containers
*out = make([]ContainerOverride, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
if in.SchedulingGates != nil {
in, out := &in.SchedulingGates, &out.SchedulingGates
*out = make([]v1.PodSchedulingGate, len(*in))
copy(*out, *in)
}
if in.ImagePullSecrets != nil {
in, out := &in.ImagePullSecrets, &out.ImagePullSecrets
*out = make([]v1.LocalObjectReference, len(*in))
copy(*out, *in)
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodSpecPatch.
func (in *PodSpecPatch) DeepCopy() *PodSpecPatch {
if in == nil {
return nil
}
out := new(PodSpecPatch)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *PodTemplatePatch) DeepCopyInto(out *PodTemplatePatch) {
*out = *in
if in.Metadata != nil {
in, out := &in.Metadata, &out.Metadata
*out = new(metav1.ObjectMeta)
(*in).DeepCopyInto(*out)
}
if in.Spec != nil {
in, out := &in.Spec, &out.Spec
*out = new(PodSpecPatch)
(*in).DeepCopyInto(*out)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodTemplateSpecOverride.
func (in *PodTemplateSpecOverride) DeepCopy() *PodTemplateSpecOverride {
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodTemplatePatch.
func (in *PodTemplatePatch) DeepCopy() *PodTemplatePatch {
if in == nil {
return nil
}
out := new(PodTemplateSpecOverride)
out := new(PodTemplatePatch)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *ReplicatedJobPatch) DeepCopyInto(out *ReplicatedJobPatch) {
*out = *in
if in.Template != nil {
in, out := &in.Template, &out.Template
*out = new(JobTemplatePatch)
(*in).DeepCopyInto(*out)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ReplicatedJobPatch.
func (in *ReplicatedJobPatch) DeepCopy() *ReplicatedJobPatch {
if in == nil {
return nil
}
out := new(ReplicatedJobPatch)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *RuntimePatch) DeepCopyInto(out *RuntimePatch) {
*out = *in
if in.Time != nil {
in, out := &in.Time, &out.Time
*out = (*in).DeepCopy()
}
if in.TrainingRuntimeSpec != nil {
in, out := &in.TrainingRuntimeSpec, &out.TrainingRuntimeSpec
*out = new(TrainingRuntimeSpecPatch)
(*in).DeepCopyInto(*out)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimePatch.
func (in *RuntimePatch) DeepCopy() *RuntimePatch {
if in == nil {
return nil
}
out := new(RuntimePatch)
in.DeepCopyInto(out)
return out
}
@@ -717,23 +833,9 @@ func (in *TrainJobSpec) DeepCopyInto(out *TrainJobSpec) {
*out = new(Trainer)
(*in).DeepCopyInto(*out)
}
if in.Labels != nil {
in, out := &in.Labels, &out.Labels
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
if in.Annotations != nil {
in, out := &in.Annotations, &out.Annotations
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
if in.PodTemplateOverrides != nil {
in, out := &in.PodTemplateOverrides, &out.PodTemplateOverrides
*out = make([]PodTemplateOverride, len(*in))
if in.RuntimePatches != nil {
in, out := &in.RuntimePatches, &out.RuntimePatches
*out = make([]RuntimePatch, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
@@ -931,6 +1033,27 @@ func (in *TrainingRuntimeSpec) DeepCopy() *TrainingRuntimeSpec {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TrainingRuntimeSpecPatch) DeepCopyInto(out *TrainingRuntimeSpecPatch) {
*out = *in
if in.Template != nil {
in, out := &in.Template, &out.Template
*out = new(JobSetTemplatePatch)
(*in).DeepCopyInto(*out)
}
return
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TrainingRuntimeSpecPatch.
func (in *TrainingRuntimeSpecPatch) DeepCopy() *TrainingRuntimeSpecPatch {
if in == nil {
return nil
}
out := new(TrainingRuntimeSpecPatch)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *VolcanoPodGroupPolicySource) DeepCopyInto(out *VolcanoPodGroupPolicySource) {
*out = *in
pkg/apis/trainer/v1alpha1/zz_generated.defaults.go
Generated
+86 -71
View File
@@ -269,77 +269,92 @@ func SetObjectDefaults_TrainJob(in *TrainJob) {
}
}
}
for i := range in.Spec.PodTemplateOverrides {
a := &in.Spec.PodTemplateOverrides[i]
if a.Spec != nil {
for j := range a.Spec.Volumes {
b := &a.Spec.Volumes[j]
if b.VolumeSource.ISCSI != nil {
if b.VolumeSource.ISCSI.ISCSIInterface == "" {
b.VolumeSource.ISCSI.ISCSIInterface = "default"
}
}
if b.VolumeSource.RBD != nil {
if b.VolumeSource.RBD.RBDPool == "" {
b.VolumeSource.RBD.RBDPool = "rbd"
}
if b.VolumeSource.RBD.RadosUser == "" {
b.VolumeSource.RBD.RadosUser = "admin"
}
if b.VolumeSource.RBD.Keyring == "" {
b.VolumeSource.RBD.Keyring = "/etc/ceph/keyring"
}
}
if b.VolumeSource.AzureDisk != nil {
if b.VolumeSource.AzureDisk.CachingMode == nil {
ptrVar1 := v1.AzureDataDiskCachingMode(v1.AzureDataDiskCachingReadWrite)
b.VolumeSource.AzureDisk.CachingMode = &ptrVar1
}
if b.VolumeSource.AzureDisk.FSType == nil {
var ptrVar1 string = "ext4"
b.VolumeSource.AzureDisk.FSType = &ptrVar1
}
if b.VolumeSource.AzureDisk.ReadOnly == nil {
var ptrVar1 bool = false
b.VolumeSource.AzureDisk.ReadOnly = &ptrVar1
}
if b.VolumeSource.AzureDisk.Kind == nil {
ptrVar1 := v1.AzureDataDiskKind(v1.AzureSharedBlobDisk)
b.VolumeSource.AzureDisk.Kind = &ptrVar1
}
}
if b.VolumeSource.ScaleIO != nil {
if b.VolumeSource.ScaleIO.StorageMode == "" {
b.VolumeSource.ScaleIO.StorageMode = "ThinProvisioned"
}
if b.VolumeSource.ScaleIO.FSType == "" {
b.VolumeSource.ScaleIO.FSType = "xfs"
}
}
}
for j := range a.Spec.InitContainers {
b := &a.Spec.InitContainers[j]
for k := range b.Env {
c := &b.Env[k]
if c.ValueFrom != nil {
if c.ValueFrom.FileKeyRef != nil {
if c.ValueFrom.FileKeyRef.Optional == nil {
var ptrVar1 bool = false
c.ValueFrom.FileKeyRef.Optional = &ptrVar1
}
}
}
}
}
for j := range a.Spec.Containers {
b := &a.Spec.Containers[j]
for k := range b.Env {
c := &b.Env[k]
if c.ValueFrom != nil {
if c.ValueFrom.FileKeyRef != nil {
if c.ValueFrom.FileKeyRef.Optional == nil {
var ptrVar1 bool = false
c.ValueFrom.FileKeyRef.Optional = &ptrVar1
for i := range in.Spec.RuntimePatches {
a := &in.Spec.RuntimePatches[i]
if a.TrainingRuntimeSpec != nil {
if a.TrainingRuntimeSpec.Template != nil {
if a.TrainingRuntimeSpec.Template.Spec != nil {
for j := range a.TrainingRuntimeSpec.Template.Spec.ReplicatedJobs {
b := &a.TrainingRuntimeSpec.Template.Spec.ReplicatedJobs[j]
if b.Template != nil {
if b.Template.Spec != nil {
if b.Template.Spec.Template != nil {
if b.Template.Spec.Template.Spec != nil {
for k := range b.Template.Spec.Template.Spec.Volumes {
c := &b.Template.Spec.Template.Spec.Volumes[k]
if c.VolumeSource.ISCSI != nil {
if c.VolumeSource.ISCSI.ISCSIInterface == "" {
c.VolumeSource.ISCSI.ISCSIInterface = "default"
}
}
if c.VolumeSource.RBD != nil {
if c.VolumeSource.RBD.RBDPool == "" {
c.VolumeSource.RBD.RBDPool = "rbd"
}
if c.VolumeSource.RBD.RadosUser == "" {
c.VolumeSource.RBD.RadosUser = "admin"
}
if c.VolumeSource.RBD.Keyring == "" {
c.VolumeSource.RBD.Keyring = "/etc/ceph/keyring"
}
}
if c.VolumeSource.AzureDisk != nil {
if c.VolumeSource.AzureDisk.CachingMode == nil {
ptrVar1 := v1.AzureDataDiskCachingMode(v1.AzureDataDiskCachingReadWrite)
c.VolumeSource.AzureDisk.CachingMode = &ptrVar1
}
if c.VolumeSource.AzureDisk.FSType == nil {
var ptrVar1 string = "ext4"
c.VolumeSource.AzureDisk.FSType = &ptrVar1
}
if c.VolumeSource.AzureDisk.ReadOnly == nil {
var ptrVar1 bool = false
c.VolumeSource.AzureDisk.ReadOnly = &ptrVar1
}
if c.VolumeSource.AzureDisk.Kind == nil {
ptrVar1 := v1.AzureDataDiskKind(v1.AzureSharedBlobDisk)
c.VolumeSource.AzureDisk.Kind = &ptrVar1
}
}
if c.VolumeSource.ScaleIO != nil {
if c.VolumeSource.ScaleIO.StorageMode == "" {
c.VolumeSource.ScaleIO.StorageMode = "ThinProvisioned"
}
if c.VolumeSource.ScaleIO.FSType == "" {
c.VolumeSource.ScaleIO.FSType = "xfs"
}
}
}
for k := range b.Template.Spec.Template.Spec.InitContainers {
c := &b.Template.Spec.Template.Spec.InitContainers[k]
for l := range c.Env {
d := &c.Env[l]
if d.ValueFrom != nil {
if d.ValueFrom.FileKeyRef != nil {
if d.ValueFrom.FileKeyRef.Optional == nil {
var ptrVar1 bool = false
d.ValueFrom.FileKeyRef.Optional = &ptrVar1
}
}
}
}
}
for k := range b.Template.Spec.Template.Spec.Containers {
c := &b.Template.Spec.Template.Spec.Containers[k]
for l := range c.Env {
d := &c.Env[l]
if d.ValueFrom != nil {
if d.ValueFrom.FileKeyRef != nil {
if d.ValueFrom.FileKeyRef.Optional == nil {
var ptrVar1 bool = false
d.ValueFrom.FileKeyRef.Optional = &ptrVar1
}
}
}
}
}
}
}
}
}
}
pkg/apis/trainer/v1alpha1/zz_generated.openapi.go
Generated
+326 -199
View File
@@ -35,23 +35,28 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA
return map[string]common.OpenAPIDefinition{
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.ClusterTrainingRuntime": schema_pkg_apis_trainer_v1alpha1_ClusterTrainingRuntime(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.ClusterTrainingRuntimeList": schema_pkg_apis_trainer_v1alpha1_ClusterTrainingRuntimeList(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.ContainerOverride": schema_pkg_apis_trainer_v1alpha1_ContainerOverride(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.ContainerPatch": schema_pkg_apis_trainer_v1alpha1_ContainerPatch(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.CoschedulingPodGroupPolicySource": schema_pkg_apis_trainer_v1alpha1_CoschedulingPodGroupPolicySource(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.DatasetInitializer": schema_pkg_apis_trainer_v1alpha1_DatasetInitializer(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.FluxMLPolicySource": schema_pkg_apis_trainer_v1alpha1_FluxMLPolicySource(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.Initializer": schema_pkg_apis_trainer_v1alpha1_Initializer(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.JAXMLPolicySource": schema_pkg_apis_trainer_v1alpha1_JAXMLPolicySource(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.JobSetSpecPatch": schema_pkg_apis_trainer_v1alpha1_JobSetSpecPatch(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.JobSetTemplatePatch": schema_pkg_apis_trainer_v1alpha1_JobSetTemplatePatch(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.JobSetTemplateSpec": schema_pkg_apis_trainer_v1alpha1_JobSetTemplateSpec(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.JobSpecPatch": schema_pkg_apis_trainer_v1alpha1_JobSpecPatch(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.JobStatus": schema_pkg_apis_trainer_v1alpha1_JobStatus(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.JobTemplatePatch": schema_pkg_apis_trainer_v1alpha1_JobTemplatePatch(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.MLPolicy": schema_pkg_apis_trainer_v1alpha1_MLPolicy(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.MLPolicySource": schema_pkg_apis_trainer_v1alpha1_MLPolicySource(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.MPIMLPolicySource": schema_pkg_apis_trainer_v1alpha1_MPIMLPolicySource(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.ModelInitializer": schema_pkg_apis_trainer_v1alpha1_ModelInitializer(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodGroupPolicy": schema_pkg_apis_trainer_v1alpha1_PodGroupPolicy(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodGroupPolicySource": schema_pkg_apis_trainer_v1alpha1_PodGroupPolicySource(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplateOverride": schema_pkg_apis_trainer_v1alpha1_PodTemplateOverride(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplateOverrideTargetJob": schema_pkg_apis_trainer_v1alpha1_PodTemplateOverrideTargetJob(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplateSpecOverride": schema_pkg_apis_trainer_v1alpha1_PodTemplateSpecOverride(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodSpecPatch": schema_pkg_apis_trainer_v1alpha1_PodSpecPatch(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplatePatch": schema_pkg_apis_trainer_v1alpha1_PodTemplatePatch(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.ReplicatedJobPatch": schema_pkg_apis_trainer_v1alpha1_ReplicatedJobPatch(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.RuntimePatch": schema_pkg_apis_trainer_v1alpha1_RuntimePatch(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.RuntimeRef": schema_pkg_apis_trainer_v1alpha1_RuntimeRef(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TorchMLPolicySource": schema_pkg_apis_trainer_v1alpha1_TorchMLPolicySource(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TrainJob": schema_pkg_apis_trainer_v1alpha1_TrainJob(ref),
@@ -62,6 +67,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TrainingRuntime": schema_pkg_apis_trainer_v1alpha1_TrainingRuntime(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TrainingRuntimeList": schema_pkg_apis_trainer_v1alpha1_TrainingRuntimeList(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TrainingRuntimeSpec": schema_pkg_apis_trainer_v1alpha1_TrainingRuntimeSpec(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TrainingRuntimeSpecPatch": schema_pkg_apis_trainer_v1alpha1_TrainingRuntimeSpecPatch(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.VolcanoPodGroupPolicySource": schema_pkg_apis_trainer_v1alpha1_VolcanoPodGroupPolicySource(ref),
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.XGBoostMLPolicySource": schema_pkg_apis_trainer_v1alpha1_XGBoostMLPolicySource(ref),
v2.ContainerResourceMetricSource{}.OpenAPIModelName(): schema_k8sio_api_autoscaling_v2_ContainerResourceMetricSource(ref),
@@ -524,16 +530,16 @@ func schema_pkg_apis_trainer_v1alpha1_ClusterTrainingRuntimeList(ref common.Refe
}
}
func schema_pkg_apis_trainer_v1alpha1_ContainerOverride(ref common.ReferenceCallback) common.OpenAPIDefinition {
func schema_pkg_apis_trainer_v1alpha1_ContainerPatch(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
SchemaProps: spec.SchemaProps{
Description: "ContainerOverride represents parameters that can be overridden using PodSpecOverrides.",
Description: "ContainerPatch represents parameters that can be patched using PodSpecPatch.",
Type: []string{"object"},
Properties: map[string]spec.Schema{
"name": {
SchemaProps: spec.SchemaProps{
Description: "name for the container. TrainingRuntime must have this container.",
Description: "name for the container. Runtime must have this container.",
Type: []string{"string"},
Format: "",
},
@@ -548,7 +554,7 @@ func schema_pkg_apis_trainer_v1alpha1_ContainerOverride(ref common.ReferenceCall
},
},
SchemaProps: spec.SchemaProps{
Description: "env is the list of environment variables to set in the container. These values will be merged with the TrainingRuntime's environments. These values can't be set for container with the name: `node`, `dataset-initializer`, or `model-initializer`. For those containers the envs can only be set via Trainer or Initializer APIs.",
Description: "env is the list of environment variables to set in the container. These values will be merged with the Runtime's environments. These values can't be set for container with the name: `node`, `dataset-initializer`, or `model-initializer`. For those containers the envs can only be set via Trainer or Initializer APIs.",
Type: []string{"array"},
Items: &spec.SchemaOrArray{
Schema: &spec.Schema{
@@ -584,7 +590,7 @@ func schema_pkg_apis_trainer_v1alpha1_ContainerOverride(ref common.ReferenceCall
},
"securityContext": {
SchemaProps: spec.SchemaProps{
Description: "securityContext overrides the container's security context. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/",
Description: "securityContext patches the container's security context. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/",
Ref: ref(corev1.SecurityContext{}.OpenAPIModelName()),
},
},
@@ -725,6 +731,70 @@ func schema_pkg_apis_trainer_v1alpha1_JAXMLPolicySource(ref common.ReferenceCall
}
}
func schema_pkg_apis_trainer_v1alpha1_JobSetSpecPatch(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
SchemaProps: spec.SchemaProps{
Description: "JobSetSpecPatch defines allowed patches for the JobSet spec.",
Type: []string{"object"},
Properties: map[string]spec.Schema{
"replicatedJobs": {
VendorExtensible: spec.VendorExtensible{
Extensions: spec.Extensions{
"x-kubernetes-list-map-keys": []interface{}{
"name",
},
"x-kubernetes-list-type": "map",
},
},
SchemaProps: spec.SchemaProps{
Description: "replicatedJobs defines per-job patches, keyed by job name.",
Type: []string{"array"},
Items: &spec.SchemaOrArray{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Default: map[string]interface{}{},
Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.ReplicatedJobPatch"),
},
},
},
},
},
},
},
},
Dependencies: []string{
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.ReplicatedJobPatch"},
}
}
func schema_pkg_apis_trainer_v1alpha1_JobSetTemplatePatch(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
SchemaProps: spec.SchemaProps{
Description: "JobSetTemplatePatch defines patches for the JobSet template. It mirrors JobSetTemplateSpec but only exposes metadata and restricted spec fields.",
Type: []string{"object"},
Properties: map[string]spec.Schema{
"metadata": {
SchemaProps: spec.SchemaProps{
Description: "metadata patches the JobSet object metadata. Only labels and annotations are allowed.",
Ref: ref(metav1.ObjectMeta{}.OpenAPIModelName()),
},
},
"spec": {
SchemaProps: spec.SchemaProps{
Description: "spec patches the JobSet spec with restricted fields.",
Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.JobSetSpecPatch"),
},
},
},
},
},
Dependencies: []string{
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.JobSetSpecPatch", metav1.ObjectMeta{}.OpenAPIModelName()},
}
}
func schema_pkg_apis_trainer_v1alpha1_JobSetTemplateSpec(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
@@ -754,6 +824,27 @@ func schema_pkg_apis_trainer_v1alpha1_JobSetTemplateSpec(ref common.ReferenceCal
}
}
func schema_pkg_apis_trainer_v1alpha1_JobSpecPatch(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
SchemaProps: spec.SchemaProps{
Description: "JobSpecPatch defines allowed patches for the Job spec.",
Type: []string{"object"},
Properties: map[string]spec.Schema{
"template": {
SchemaProps: spec.SchemaProps{
Description: "template patches the Pod template for this Job.",
Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplatePatch"),
},
},
},
},
},
Dependencies: []string{
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplatePatch"},
}
}
func schema_pkg_apis_trainer_v1alpha1_JobStatus(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
@@ -809,6 +900,33 @@ func schema_pkg_apis_trainer_v1alpha1_JobStatus(ref common.ReferenceCallback) co
}
}
func schema_pkg_apis_trainer_v1alpha1_JobTemplatePatch(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
SchemaProps: spec.SchemaProps{
Description: "JobTemplatePatch defines patches for a Job template within a replicated job.",
Type: []string{"object"},
Properties: map[string]spec.Schema{
"metadata": {
SchemaProps: spec.SchemaProps{
Description: "metadata patches the Job template metadata. Only labels and annotations are allowed.",
Ref: ref(metav1.ObjectMeta{}.OpenAPIModelName()),
},
},
"spec": {
SchemaProps: spec.SchemaProps{
Description: "spec patches the Job spec with restricted fields.",
Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.JobSpecPatch"),
},
},
},
},
},
Dependencies: []string{
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.JobSpecPatch", metav1.ObjectMeta{}.OpenAPIModelName()},
}
}
func schema_pkg_apis_trainer_v1alpha1_MLPolicy(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
@@ -1051,141 +1169,20 @@ func schema_pkg_apis_trainer_v1alpha1_PodGroupPolicySource(ref common.ReferenceC
}
}
func schema_pkg_apis_trainer_v1alpha1_PodTemplateOverride(ref common.ReferenceCallback) common.OpenAPIDefinition {
func schema_pkg_apis_trainer_v1alpha1_PodSpecPatch(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
SchemaProps: spec.SchemaProps{
Description: "PodTemplateOverride represents a custom PodTemplateSpec override that will be applied to the TrainJob's training runtime.",
Type: []string{"object"},
Properties: map[string]spec.Schema{
"targetJobs": {
VendorExtensible: spec.VendorExtensible{
Extensions: spec.Extensions{
"x-kubernetes-list-type": "atomic",
},
},
SchemaProps: spec.SchemaProps{
Description: "targetJobs is the list of replicated jobs in the training runtime template to apply the overrides.",
Type: []string{"array"},
Items: &spec.SchemaOrArray{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Default: map[string]interface{}{},
Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplateOverrideTargetJob"),
},
},
},
},
},
"manager": {
SchemaProps: spec.SchemaProps{
Description: "manager indicates which controller created this PodTemplateOverride object. It can be used by external controllers or admission webhooks to track ownership and avoid conflicts with other controllers. For example, Kueue sets this field to \"kueue.k8s.io/manager\". Defaults to `trainer.kubeflow.org/unknown`",
Type: []string{"string"},
Format: "",
},
},
"metadata": {
SchemaProps: spec.SchemaProps{
Description: "metadata overrides the Pod template metadata. These values will be merged with the TrainingRuntime's Pod template metadata.",
Ref: ref(metav1.ObjectMeta{}.OpenAPIModelName()),
},
},
"spec": {
SchemaProps: spec.SchemaProps{
Description: "spec overrides the Pod template spec. These values will be merged with the TrainingRuntime's Pod template spec.",
Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplateSpecOverride"),
},
},
},
Required: []string{"targetJobs"},
},
},
Dependencies: []string{
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplateOverrideTargetJob", "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplateSpecOverride", metav1.ObjectMeta{}.OpenAPIModelName()},
}
}
func schema_pkg_apis_trainer_v1alpha1_PodTemplateOverrideTargetJob(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
SchemaProps: spec.SchemaProps{
Type: []string{"object"},
Properties: map[string]spec.Schema{
"name": {
SchemaProps: spec.SchemaProps{
Description: "name is the target training job name for which the PodTemplateSpec is overridden.",
Type: []string{"string"},
Format: "",
},
},
},
Required: []string{"name"},
},
},
}
}
func schema_pkg_apis_trainer_v1alpha1_PodTemplateSpecOverride(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
SchemaProps: spec.SchemaProps{
Description: "PodTemplateSpecOverride represents the spec overrides for Pod template.",
Description: "PodSpecPatch contains the Pod spec fields that managers are permitted to patch.",
Type: []string{"object"},
Properties: map[string]spec.Schema{
"serviceAccountName": {
SchemaProps: spec.SchemaProps{
Description: "serviceAccountName overrides the service account.",
Description: "serviceAccountName patches the service account for the Pods in the target job templates.",
Type: []string{"string"},
Format: "",
},
},
"nodeSelector": {
SchemaProps: spec.SchemaProps{
Description: "nodeSelector overrides the node selector to place Pod on the specific node.",
Type: []string{"object"},
AdditionalProperties: &spec.SchemaOrBool{
Allows: true,
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Default: "",
Type: []string{"string"},
Format: "",
},
},
},
},
},
"affinity": {
SchemaProps: spec.SchemaProps{
Description: "affinity overrides for the Pod's affinity.",
Ref: ref(corev1.Affinity{}.OpenAPIModelName()),
},
},
"tolerations": {
VendorExtensible: spec.VendorExtensible{
Extensions: spec.Extensions{
"x-kubernetes-list-type": "atomic",
},
},
SchemaProps: spec.SchemaProps{
Description: "tolerations overrides the Pod's tolerations.",
Type: []string{"array"},
Items: &spec.SchemaOrArray{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Default: map[string]interface{}{},
Ref: ref(corev1.Toleration{}.OpenAPIModelName()),
},
},
},
},
},
"securityContext": {
SchemaProps: spec.SchemaProps{
Description: "securityContext overrides the Pod's security context. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/",
Ref: ref(corev1.PodSecurityContext{}.OpenAPIModelName()),
},
},
"volumes": {
VendorExtensible: spec.VendorExtensible{
Extensions: spec.Extensions{
@@ -1196,7 +1193,7 @@ func schema_pkg_apis_trainer_v1alpha1_PodTemplateSpecOverride(ref common.Referen
},
},
SchemaProps: spec.SchemaProps{
Description: "volumes overrides the Pod's volumes.",
Description: "volumes patches the Pod's volumes.",
Type: []string{"array"},
Items: &spec.SchemaOrArray{
Schema: &spec.Schema{
@@ -1218,13 +1215,13 @@ func schema_pkg_apis_trainer_v1alpha1_PodTemplateSpecOverride(ref common.Referen
},
},
SchemaProps: spec.SchemaProps{
Description: "initContainers overrides the init container in the target job templates.",
Description: "initContainers patches the init containers in the target job templates.",
Type: []string{"array"},
Items: &spec.SchemaOrArray{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Default: map[string]interface{}{},
Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.ContainerOverride"),
Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.ContainerPatch"),
},
},
},
@@ -1240,35 +1237,13 @@ func schema_pkg_apis_trainer_v1alpha1_PodTemplateSpecOverride(ref common.Referen
},
},
SchemaProps: spec.SchemaProps{
Description: "containers overrides for the containers in the target job templates.",
Description: "containers patches specific containers in the target job templates.",
Type: []string{"array"},
Items: &spec.SchemaOrArray{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Default: map[string]interface{}{},
Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.ContainerOverride"),
},
},
},
},
},
"schedulingGates": {
VendorExtensible: spec.VendorExtensible{
Extensions: spec.Extensions{
"x-kubernetes-list-map-keys": []interface{}{
"name",
},
"x-kubernetes-list-type": "map",
},
},
SchemaProps: spec.SchemaProps{
Description: "schedulingGates overrides the scheduling gates of the Pods in the target job templates. More info: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-scheduling-readiness/",
Type: []string{"array"},
Items: &spec.SchemaOrArray{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Default: map[string]interface{}{},
Ref: ref(corev1.PodSchedulingGate{}.OpenAPIModelName()),
Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.ContainerPatch"),
},
},
},
@@ -1284,7 +1259,7 @@ func schema_pkg_apis_trainer_v1alpha1_PodTemplateSpecOverride(ref common.Referen
},
},
SchemaProps: spec.SchemaProps{
Description: "imagePullSecrets overrides the image pull secrets for the Pods in the target job templates.",
Description: "imagePullSecrets patches the image pull secrets for the Pods in the target job templates.",
Type: []string{"array"},
Items: &spec.SchemaOrArray{
Schema: &spec.Schema{
@@ -1296,11 +1271,171 @@ func schema_pkg_apis_trainer_v1alpha1_PodTemplateSpecOverride(ref common.Referen
},
},
},
"securityContext": {
SchemaProps: spec.SchemaProps{
Description: "securityContext patches the Pod's security context. More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/",
Ref: ref(corev1.PodSecurityContext{}.OpenAPIModelName()),
},
},
"nodeSelector": {
SchemaProps: spec.SchemaProps{
Description: "nodeSelector patches the node selector to place Pods on specific nodes.",
Type: []string{"object"},
AdditionalProperties: &spec.SchemaOrBool{
Allows: true,
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Default: "",
Type: []string{"string"},
Format: "",
},
},
},
},
},
"affinity": {
SchemaProps: spec.SchemaProps{
Description: "affinity patches the Pod's scheduling affinity.",
Ref: ref(corev1.Affinity{}.OpenAPIModelName()),
},
},
"tolerations": {
VendorExtensible: spec.VendorExtensible{
Extensions: spec.Extensions{
"x-kubernetes-list-type": "atomic",
},
},
SchemaProps: spec.SchemaProps{
Description: "tolerations patches the Pod's tolerations.",
Type: []string{"array"},
Items: &spec.SchemaOrArray{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Default: map[string]interface{}{},
Ref: ref(corev1.Toleration{}.OpenAPIModelName()),
},
},
},
},
},
"schedulingGates": {
VendorExtensible: spec.VendorExtensible{
Extensions: spec.Extensions{
"x-kubernetes-list-map-keys": []interface{}{
"name",
},
"x-kubernetes-list-type": "map",
},
},
SchemaProps: spec.SchemaProps{
Description: "schedulingGates patches the scheduling gates for the Pods in the target job templates. More info: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-scheduling-readiness/",
Type: []string{"array"},
Items: &spec.SchemaOrArray{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Default: map[string]interface{}{},
Ref: ref(corev1.PodSchedulingGate{}.OpenAPIModelName()),
},
},
},
},
},
},
},
},
Dependencies: []string{
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.ContainerOverride", corev1.Affinity{}.OpenAPIModelName(), corev1.LocalObjectReference{}.OpenAPIModelName(), corev1.PodSchedulingGate{}.OpenAPIModelName(), corev1.PodSecurityContext{}.OpenAPIModelName(), corev1.Toleration{}.OpenAPIModelName(), corev1.Volume{}.OpenAPIModelName()},
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.ContainerPatch", corev1.Affinity{}.OpenAPIModelName(), corev1.LocalObjectReference{}.OpenAPIModelName(), corev1.PodSchedulingGate{}.OpenAPIModelName(), corev1.PodSecurityContext{}.OpenAPIModelName(), corev1.Toleration{}.OpenAPIModelName(), corev1.Volume{}.OpenAPIModelName()},
}
}
func schema_pkg_apis_trainer_v1alpha1_PodTemplatePatch(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
SchemaProps: spec.SchemaProps{
Description: "PodTemplatePatch defines patches for a Pod template within a Job.",
Type: []string{"object"},
Properties: map[string]spec.Schema{
"metadata": {
SchemaProps: spec.SchemaProps{
Description: "metadata patches the Pod template metadata. Only labels and annotations are allowed.",
Ref: ref(metav1.ObjectMeta{}.OpenAPIModelName()),
},
},
"spec": {
SchemaProps: spec.SchemaProps{
Description: "spec patches the Pod spec with the fields managers are permitted to set.",
Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodSpecPatch"),
},
},
},
},
},
Dependencies: []string{
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodSpecPatch", metav1.ObjectMeta{}.OpenAPIModelName()},
}
}
func schema_pkg_apis_trainer_v1alpha1_ReplicatedJobPatch(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
SchemaProps: spec.SchemaProps{
Description: "ReplicatedJobPatch defines patches for a specific replicated job within the JobSet.",
Type: []string{"object"},
Properties: map[string]spec.Schema{
"name": {
SchemaProps: spec.SchemaProps{
Description: "name is the name of the replicated job to patch.",
Type: []string{"string"},
Format: "",
},
},
"template": {
SchemaProps: spec.SchemaProps{
Description: "template patches the Job template for this replicated job.",
Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.JobTemplatePatch"),
},
},
},
Required: []string{"name"},
},
},
Dependencies: []string{
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.JobTemplatePatch"},
}
}
func schema_pkg_apis_trainer_v1alpha1_RuntimePatch(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
SchemaProps: spec.SchemaProps{
Description: "RuntimePatch represents a custom patch applied to the TrainJob's training runtime template. Patches are keyed by manager to provide clear ownership and avoid conflicts between controllers.",
Type: []string{"object"},
Properties: map[string]spec.Schema{
"manager": {
SchemaProps: spec.SchemaProps{
Description: "manager indicates who owns this patch entry. It can be set by the user, external controllers, or admission webhooks to track ownership and avoid conflicts. For example, Kueue sets this field to \"kueue.x-k8s.io/manager\".",
Type: []string{"string"},
Format: "",
},
},
"time": {
SchemaProps: spec.SchemaProps{
Description: "time is the timestamp of when this patch was last written. Set by the Trainer admission webhook on each create or update. Used for observability only, not used as a list map key.",
Ref: ref(metav1.Time{}.OpenAPIModelName()),
},
},
"trainingRuntimeSpec": {
SchemaProps: spec.SchemaProps{
Description: "trainingRuntimeSpec defines allowed patches for ClusterTrainingRuntime or TrainingRuntime-based jobs.",
Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TrainingRuntimeSpecPatch"),
},
},
},
Required: []string{"manager"},
},
},
Dependencies: []string{
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.TrainingRuntimeSpecPatch", metav1.Time{}.OpenAPIModelName()},
}
}
@@ -1477,52 +1612,23 @@ func schema_pkg_apis_trainer_v1alpha1_TrainJobSpec(ref common.ReferenceCallback)
Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.Trainer"),
},
},
"labels": {
SchemaProps: spec.SchemaProps{
Description: "labels to apply for the derivative JobSet and Jobs. They will be merged with the TrainingRuntime values.",
Type: []string{"object"},
AdditionalProperties: &spec.SchemaOrBool{
Allows: true,
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Default: "",
Type: []string{"string"},
Format: "",
},
},
},
},
},
"annotations": {
SchemaProps: spec.SchemaProps{
Description: "annotations to apply for the derivative JobSet and Jobs. They will be merged with the TrainingRuntime values.",
Type: []string{"object"},
AdditionalProperties: &spec.SchemaOrBool{
Allows: true,
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Default: "",
Type: []string{"string"},
Format: "",
},
},
},
},
},
"podTemplateOverrides": {
"runtimePatches": {
VendorExtensible: spec.VendorExtensible{
Extensions: spec.Extensions{
"x-kubernetes-list-type": "atomic",
"x-kubernetes-list-map-keys": []interface{}{
"manager",
},
"x-kubernetes-list-type": "map",
},
},
SchemaProps: spec.SchemaProps{
Description: "podTemplateOverrides define the PodTemplateOverrides for the training runtime. When multiple overrides apply to the same targetJob, later entries in the slice override earlier field values.",
Description: "runtimePatches defines custom patches applied to the TrainJob's Runtime. Patches are keyed by manager to provide clear ownership and avoid conflicts between controllers.",
Type: []string{"array"},
Items: &spec.SchemaOrArray{
Schema: &spec.Schema{
SchemaProps: spec.SchemaProps{
Default: map[string]interface{}{},
Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplateOverride"),
Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.RuntimePatch"),
},
},
},
@@ -1547,7 +1653,7 @@ func schema_pkg_apis_trainer_v1alpha1_TrainJobSpec(ref common.ReferenceCallback)
},
},
Dependencies: []string{
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.Initializer", "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.PodTemplateOverride", "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.RuntimeRef", "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.Trainer"},
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.Initializer", "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.RuntimePatch", "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.RuntimeRef", "github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.Trainer"},
}
}
@@ -1842,6 +1948,27 @@ func schema_pkg_apis_trainer_v1alpha1_TrainingRuntimeSpec(ref common.ReferenceCa
}
}
func schema_pkg_apis_trainer_v1alpha1_TrainingRuntimeSpecPatch(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
SchemaProps: spec.SchemaProps{
Description: "TrainingRuntimeSpecPatch mirrors TrainingRuntimeSpec but only exposes the fields managers are permitted to patch.",
Type: []string{"object"},
Properties: map[string]spec.Schema{
"template": {
SchemaProps: spec.SchemaProps{
Description: "template patches the JobSet template.",
Ref: ref("github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.JobSetTemplatePatch"),
},
},
},
},
},
Dependencies: []string{
"github.com/kubeflow/trainer/v2/pkg/apis/trainer/v1alpha1.JobSetTemplatePatch"},
}
}
func schema_pkg_apis_trainer_v1alpha1_VolcanoPodGroupPolicySource(ref common.ReferenceCallback) common.OpenAPIDefinition {
return common.OpenAPIDefinition{
Schema: spec.Schema{
pkg/client/applyconfiguration/trainer/v1alpha1/containeroverride.go → pkg/client/applyconfiguration/trainer/v1alpha1/containerpatch.go
Generated
+13 -13
View File
@@ -21,35 +21,35 @@ import (
v1 "k8s.io/client-go/applyconfigurations/core/v1"
)
// ContainerOverrideApplyConfiguration represents a declarative configuration of the ContainerOverride type for use
// ContainerPatchApplyConfiguration represents a declarative configuration of the ContainerPatch type for use
// with apply.
//
// ContainerOverride represents parameters that can be overridden using PodSpecOverrides.
type ContainerOverrideApplyConfiguration struct {
// name for the container. TrainingRuntime must have this container.
// ContainerPatch represents parameters that can be patched using PodSpecPatch.
type ContainerPatchApplyConfiguration struct {
// name for the container. Runtime must have this container.
Name *string `json:"name,omitempty"`
// env is the list of environment variables to set in the container.
// These values will be merged with the TrainingRuntime's environments.
// These values will be merged with the Runtime's environments.
// These values can't be set for container with the name: `node`, `dataset-initializer`, or
// `model-initializer`. For those containers the envs can only be set via Trainer or Initializer APIs.
Env []v1.EnvVarApplyConfiguration `json:"env,omitempty"`
// volumeMounts are the volumes to mount into the container's filesystem.
VolumeMounts []v1.VolumeMountApplyConfiguration `json:"volumeMounts,omitempty"`
// securityContext overrides the container's security context.
// securityContext patches the container's security context.
// More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"`
}
// ContainerOverrideApplyConfiguration constructs a declarative configuration of the ContainerOverride type for use with
// ContainerPatchApplyConfiguration constructs a declarative configuration of the ContainerPatch type for use with
// apply.
func ContainerOverride() *ContainerOverrideApplyConfiguration {
return &ContainerOverrideApplyConfiguration{}
func ContainerPatch() *ContainerPatchApplyConfiguration {
return &ContainerPatchApplyConfiguration{}
}
// WithName sets the Name field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Name field is set to the value of the last call.
func (b *ContainerOverrideApplyConfiguration) WithName(value string) *ContainerOverrideApplyConfiguration {
func (b *ContainerPatchApplyConfiguration) WithName(value string) *ContainerPatchApplyConfiguration {
b.Name = &value
return b
}
@@ -57,7 +57,7 @@ func (b *ContainerOverrideApplyConfiguration) WithName(value string) *ContainerO
// WithEnv adds the given value to the Env field in the declarative configuration
// and returns the receiver, so that objects can be build by chaining "With" function invocations.
// If called multiple times, values provided by each call will be appended to the Env field.
func (b *ContainerOverrideApplyConfiguration) WithEnv(values ...*v1.EnvVarApplyConfiguration) *ContainerOverrideApplyConfiguration {
func (b *ContainerPatchApplyConfiguration) WithEnv(values ...*v1.EnvVarApplyConfiguration) *ContainerPatchApplyConfiguration {
for i := range values {
if values[i] == nil {
panic("nil value passed to WithEnv")
@@ -70,7 +70,7 @@ func (b *ContainerOverrideApplyConfiguration) WithEnv(values ...*v1.EnvVarApplyC
// WithVolumeMounts adds the given value to the VolumeMounts field in the declarative configuration
// and returns the receiver, so that objects can be build by chaining "With" function invocations.
// If called multiple times, values provided by each call will be appended to the VolumeMounts field.
func (b *ContainerOverrideApplyConfiguration) WithVolumeMounts(values ...*v1.VolumeMountApplyConfiguration) *ContainerOverrideApplyConfiguration {
func (b *ContainerPatchApplyConfiguration) WithVolumeMounts(values ...*v1.VolumeMountApplyConfiguration) *ContainerPatchApplyConfiguration {
for i := range values {
if values[i] == nil {
panic("nil value passed to WithVolumeMounts")
@@ -83,7 +83,7 @@ func (b *ContainerOverrideApplyConfiguration) WithVolumeMounts(values ...*v1.Vol
// WithSecurityContext sets the SecurityContext field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the SecurityContext field is set to the value of the last call.
func (b *ContainerOverrideApplyConfiguration) WithSecurityContext(value corev1.SecurityContext) *ContainerOverrideApplyConfiguration {
func (b *ContainerPatchApplyConfiguration) WithSecurityContext(value corev1.SecurityContext) *ContainerPatchApplyConfiguration {
b.SecurityContext = &value
return b
}
pkg/client/applyconfiguration/trainer/v1alpha1/jobsetspecpatch.go
Generated
+45
View File
@@ -0,0 +1,45 @@
// Copyright 2024 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by applyconfiguration-gen. DO NOT EDIT.
package v1alpha1
// JobSetSpecPatchApplyConfiguration represents a declarative configuration of the JobSetSpecPatch type for use
// with apply.
//
// JobSetSpecPatch defines allowed patches for the JobSet spec.
type JobSetSpecPatchApplyConfiguration struct {
// replicatedJobs defines per-job patches, keyed by job name.
ReplicatedJobs []ReplicatedJobPatchApplyConfiguration `json:"replicatedJobs,omitempty"`
}
// JobSetSpecPatchApplyConfiguration constructs a declarative configuration of the JobSetSpecPatch type for use with
// apply.
func JobSetSpecPatch() *JobSetSpecPatchApplyConfiguration {
return &JobSetSpecPatchApplyConfiguration{}
}
// WithReplicatedJobs adds the given value to the ReplicatedJobs field in the declarative configuration
// and returns the receiver, so that objects can be build by chaining "With" function invocations.
// If called multiple times, values provided by each call will be appended to the ReplicatedJobs field.
func (b *JobSetSpecPatchApplyConfiguration) WithReplicatedJobs(values ...*ReplicatedJobPatchApplyConfiguration) *JobSetSpecPatchApplyConfiguration {
for i := range values {
if values[i] == nil {
panic("nil value passed to WithReplicatedJobs")
}
b.ReplicatedJobs = append(b.ReplicatedJobs, *values[i])
}
return b
}
pkg/client/applyconfiguration/trainer/v1alpha1/jobsettemplatepatch.go
Generated
+56
View File
@@ -0,0 +1,56 @@
// Copyright 2024 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by applyconfiguration-gen. DO NOT EDIT.
package v1alpha1
import (
v1 "k8s.io/client-go/applyconfigurations/meta/v1"
)
// JobSetTemplatePatchApplyConfiguration represents a declarative configuration of the JobSetTemplatePatch type for use
// with apply.
//
// JobSetTemplatePatch defines patches for the JobSet template.
// It mirrors JobSetTemplateSpec but only exposes metadata and restricted spec fields.
type JobSetTemplatePatchApplyConfiguration struct {
// metadata patches the JobSet object metadata.
// Only labels and annotations are allowed.
Metadata *v1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"`
// spec patches the JobSet spec with restricted fields.
Spec *JobSetSpecPatchApplyConfiguration `json:"spec,omitempty"`
}
// JobSetTemplatePatchApplyConfiguration constructs a declarative configuration of the JobSetTemplatePatch type for use with
// apply.
func JobSetTemplatePatch() *JobSetTemplatePatchApplyConfiguration {
return &JobSetTemplatePatchApplyConfiguration{}
}
// WithMetadata sets the Metadata field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Metadata field is set to the value of the last call.
func (b *JobSetTemplatePatchApplyConfiguration) WithMetadata(value *v1.ObjectMetaApplyConfiguration) *JobSetTemplatePatchApplyConfiguration {
b.Metadata = value
return b
}
// WithSpec sets the Spec field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Spec field is set to the value of the last call.
func (b *JobSetTemplatePatchApplyConfiguration) WithSpec(value *JobSetSpecPatchApplyConfiguration) *JobSetTemplatePatchApplyConfiguration {
b.Spec = value
return b
}
pkg/client/applyconfiguration/trainer/v1alpha1/jobspecpatch.go
Generated
+40
View File
@@ -0,0 +1,40 @@
// Copyright 2024 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by applyconfiguration-gen. DO NOT EDIT.
package v1alpha1
// JobSpecPatchApplyConfiguration represents a declarative configuration of the JobSpecPatch type for use
// with apply.
//
// JobSpecPatch defines allowed patches for the Job spec.
type JobSpecPatchApplyConfiguration struct {
// template patches the Pod template for this Job.
Template *PodTemplatePatchApplyConfiguration `json:"template,omitempty"`
}
// JobSpecPatchApplyConfiguration constructs a declarative configuration of the JobSpecPatch type for use with
// apply.
func JobSpecPatch() *JobSpecPatchApplyConfiguration {
return &JobSpecPatchApplyConfiguration{}
}
// WithTemplate sets the Template field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Template field is set to the value of the last call.
func (b *JobSpecPatchApplyConfiguration) WithTemplate(value *PodTemplatePatchApplyConfiguration) *JobSpecPatchApplyConfiguration {
b.Template = value
return b
}
pkg/client/applyconfiguration/trainer/v1alpha1/jobtemplatepatch.go
Generated
+55
View File
@@ -0,0 +1,55 @@
// Copyright 2024 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by applyconfiguration-gen. DO NOT EDIT.
package v1alpha1
import (
v1 "k8s.io/client-go/applyconfigurations/meta/v1"
)
// JobTemplatePatchApplyConfiguration represents a declarative configuration of the JobTemplatePatch type for use
// with apply.
//
// JobTemplatePatch defines patches for a Job template within a replicated job.
type JobTemplatePatchApplyConfiguration struct {
// metadata patches the Job template metadata.
// Only labels and annotations are allowed.
Metadata *v1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"`
// spec patches the Job spec with restricted fields.
Spec *JobSpecPatchApplyConfiguration `json:"spec,omitempty"`
}
// JobTemplatePatchApplyConfiguration constructs a declarative configuration of the JobTemplatePatch type for use with
// apply.
func JobTemplatePatch() *JobTemplatePatchApplyConfiguration {
return &JobTemplatePatchApplyConfiguration{}
}
// WithMetadata sets the Metadata field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Metadata field is set to the value of the last call.
func (b *JobTemplatePatchApplyConfiguration) WithMetadata(value *v1.ObjectMetaApplyConfiguration) *JobTemplatePatchApplyConfiguration {
b.Metadata = value
return b
}
// WithSpec sets the Spec field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Spec field is set to the value of the last call.
func (b *JobTemplatePatchApplyConfiguration) WithSpec(value *JobSpecPatchApplyConfiguration) *JobTemplatePatchApplyConfiguration {
b.Spec = value
return b
}
pkg/client/applyconfiguration/trainer/v1alpha1/podtemplatespecoverride.go → pkg/client/applyconfiguration/trainer/v1alpha1/podspecpatch.go
Generated
+85 -85
View File
@@ -17,100 +17,57 @@
package v1alpha1
import (
v1 "k8s.io/api/core/v1"
corev1 "k8s.io/client-go/applyconfigurations/core/v1"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/client-go/applyconfigurations/core/v1"
)
// PodTemplateSpecOverrideApplyConfiguration represents a declarative configuration of the PodTemplateSpecOverride type for use
// PodSpecPatchApplyConfiguration represents a declarative configuration of the PodSpecPatch type for use
// with apply.
//
// PodTemplateSpecOverride represents the spec overrides for Pod template.
type PodTemplateSpecOverrideApplyConfiguration struct {
// serviceAccountName overrides the service account.
// PodSpecPatch contains the Pod spec fields that managers are permitted to patch.
type PodSpecPatchApplyConfiguration struct {
// serviceAccountName patches the service account for the Pods in the target job templates.
ServiceAccountName *string `json:"serviceAccountName,omitempty"`
// nodeSelector overrides the node selector to place Pod on the specific node.
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
// affinity overrides for the Pod's affinity.
Affinity *v1.Affinity `json:"affinity,omitempty"`
// tolerations overrides the Pod's tolerations.
Tolerations []corev1.TolerationApplyConfiguration `json:"tolerations,omitempty"`
// securityContext overrides the Pod's security context.
// volumes patches the Pod's volumes.
Volumes []v1.VolumeApplyConfiguration `json:"volumes,omitempty"`
// initContainers patches the init containers in the target job templates.
InitContainers []ContainerPatchApplyConfiguration `json:"initContainers,omitempty"`
// containers patches specific containers in the target job templates.
Containers []ContainerPatchApplyConfiguration `json:"containers,omitempty"`
// imagePullSecrets patches the image pull secrets for the Pods in the target job templates.
ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
// securityContext patches the Pod's security context.
// More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
SecurityContext *v1.PodSecurityContext `json:"securityContext,omitempty"`
// volumes overrides the Pod's volumes.
Volumes []corev1.VolumeApplyConfiguration `json:"volumes,omitempty"`
// initContainers overrides the init container in the target job templates.
InitContainers []ContainerOverrideApplyConfiguration `json:"initContainers,omitempty"`
// containers overrides for the containers in the target job templates.
Containers []ContainerOverrideApplyConfiguration `json:"containers,omitempty"`
// schedulingGates overrides the scheduling gates of the Pods in the target job templates.
SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"`
// nodeSelector patches the node selector to place Pods on specific nodes.
NodeSelector map[string]string `json:"nodeSelector,omitempty"`
// affinity patches the Pod's scheduling affinity.
Affinity *corev1.Affinity `json:"affinity,omitempty"`
// tolerations patches the Pod's tolerations.
Tolerations []v1.TolerationApplyConfiguration `json:"tolerations,omitempty"`
// schedulingGates patches the scheduling gates for the Pods in the target job templates.
// More info: https://kubernetes.io/docs/concepts/scheduling-eviction/pod-scheduling-readiness/
SchedulingGates []v1.PodSchedulingGate `json:"schedulingGates,omitempty"`
// imagePullSecrets overrides the image pull secrets for the Pods in the target job templates.
ImagePullSecrets []v1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
SchedulingGates []corev1.PodSchedulingGate `json:"schedulingGates,omitempty"`
}
// PodTemplateSpecOverrideApplyConfiguration constructs a declarative configuration of the PodTemplateSpecOverride type for use with
// PodSpecPatchApplyConfiguration constructs a declarative configuration of the PodSpecPatch type for use with
// apply.
func PodTemplateSpecOverride() *PodTemplateSpecOverrideApplyConfiguration {
return &PodTemplateSpecOverrideApplyConfiguration{}
func PodSpecPatch() *PodSpecPatchApplyConfiguration {
return &PodSpecPatchApplyConfiguration{}
}
// WithServiceAccountName sets the ServiceAccountName field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the ServiceAccountName field is set to the value of the last call.
func (b *PodTemplateSpecOverrideApplyConfiguration) WithServiceAccountName(value string) *PodTemplateSpecOverrideApplyConfiguration {
func (b *PodSpecPatchApplyConfiguration) WithServiceAccountName(value string) *PodSpecPatchApplyConfiguration {
b.ServiceAccountName = &value
return b
}
// WithNodeSelector puts the entries into the NodeSelector field in the declarative configuration
// and returns the receiver, so that objects can be build by chaining "With" function invocations.
// If called multiple times, the entries provided by each call will be put on the NodeSelector field,
// overwriting an existing map entries in NodeSelector field with the same key.
func (b *PodTemplateSpecOverrideApplyConfiguration) WithNodeSelector(entries map[string]string) *PodTemplateSpecOverrideApplyConfiguration {
if b.NodeSelector == nil && len(entries) > 0 {
b.NodeSelector = make(map[string]string, len(entries))
}
for k, v := range entries {
b.NodeSelector[k] = v
}
return b
}
// WithAffinity sets the Affinity field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Affinity field is set to the value of the last call.
func (b *PodTemplateSpecOverrideApplyConfiguration) WithAffinity(value v1.Affinity) *PodTemplateSpecOverrideApplyConfiguration {
b.Affinity = &value
return b
}
// WithTolerations adds the given value to the Tolerations field in the declarative configuration
// and returns the receiver, so that objects can be build by chaining "With" function invocations.
// If called multiple times, values provided by each call will be appended to the Tolerations field.
func (b *PodTemplateSpecOverrideApplyConfiguration) WithTolerations(values ...*corev1.TolerationApplyConfiguration) *PodTemplateSpecOverrideApplyConfiguration {
for i := range values {
if values[i] == nil {
panic("nil value passed to WithTolerations")
}
b.Tolerations = append(b.Tolerations, *values[i])
}
return b
}
// WithSecurityContext sets the SecurityContext field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the SecurityContext field is set to the value of the last call.
func (b *PodTemplateSpecOverrideApplyConfiguration) WithSecurityContext(value v1.PodSecurityContext) *PodTemplateSpecOverrideApplyConfiguration {
b.SecurityContext = &value
return b
}
// WithVolumes adds the given value to the Volumes field in the declarative configuration
// and returns the receiver, so that objects can be build by chaining "With" function invocations.
// If called multiple times, values provided by each call will be appended to the Volumes field.
func (b *PodTemplateSpecOverrideApplyConfiguration) WithVolumes(values ...*corev1.VolumeApplyConfiguration) *PodTemplateSpecOverrideApplyConfiguration {
func (b *PodSpecPatchApplyConfiguration) WithVolumes(values ...*v1.VolumeApplyConfiguration) *PodSpecPatchApplyConfiguration {
for i := range values {
if values[i] == nil {
panic("nil value passed to WithVolumes")
@@ -123,7 +80,7 @@ func (b *PodTemplateSpecOverrideApplyConfiguration) WithVolumes(values ...*corev
// WithInitContainers adds the given value to the InitContainers field in the declarative configuration
// and returns the receiver, so that objects can be build by chaining "With" function invocations.
// If called multiple times, values provided by each call will be appended to the InitContainers field.
func (b *PodTemplateSpecOverrideApplyConfiguration) WithInitContainers(values ...*ContainerOverrideApplyConfiguration) *PodTemplateSpecOverrideApplyConfiguration {
func (b *PodSpecPatchApplyConfiguration) WithInitContainers(values ...*ContainerPatchApplyConfiguration) *PodSpecPatchApplyConfiguration {
for i := range values {
if values[i] == nil {
panic("nil value passed to WithInitContainers")
@@ -136,7 +93,7 @@ func (b *PodTemplateSpecOverrideApplyConfiguration) WithInitContainers(values ..
// WithContainers adds the given value to the Containers field in the declarative configuration
// and returns the receiver, so that objects can be build by chaining "With" function invocations.
// If called multiple times, values provided by each call will be appended to the Containers field.
func (b *PodTemplateSpecOverrideApplyConfiguration) WithContainers(values ...*ContainerOverrideApplyConfiguration) *PodTemplateSpecOverrideApplyConfiguration {
func (b *PodSpecPatchApplyConfiguration) WithContainers(values ...*ContainerPatchApplyConfiguration) *PodSpecPatchApplyConfiguration {
for i := range values {
if values[i] == nil {
panic("nil value passed to WithContainers")
@@ -146,22 +103,65 @@ func (b *PodTemplateSpecOverrideApplyConfiguration) WithContainers(values ...*Co
return b
}
// WithSchedulingGates adds the given value to the SchedulingGates field in the declarative configuration
// and returns the receiver, so that objects can be build by chaining "With" function invocations.
// If called multiple times, values provided by each call will be appended to the SchedulingGates field.
func (b *PodTemplateSpecOverrideApplyConfiguration) WithSchedulingGates(values ...v1.PodSchedulingGate) *PodTemplateSpecOverrideApplyConfiguration {
for i := range values {
b.SchedulingGates = append(b.SchedulingGates, values[i])
}
return b
}
// WithImagePullSecrets adds the given value to the ImagePullSecrets field in the declarative configuration
// and returns the receiver, so that objects can be build by chaining "With" function invocations.
// If called multiple times, values provided by each call will be appended to the ImagePullSecrets field.
func (b *PodTemplateSpecOverrideApplyConfiguration) WithImagePullSecrets(values ...v1.LocalObjectReference) *PodTemplateSpecOverrideApplyConfiguration {
func (b *PodSpecPatchApplyConfiguration) WithImagePullSecrets(values ...corev1.LocalObjectReference) *PodSpecPatchApplyConfiguration {
for i := range values {
b.ImagePullSecrets = append(b.ImagePullSecrets, values[i])
}
return b
}
// WithSecurityContext sets the SecurityContext field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the SecurityContext field is set to the value of the last call.
func (b *PodSpecPatchApplyConfiguration) WithSecurityContext(value corev1.PodSecurityContext) *PodSpecPatchApplyConfiguration {
b.SecurityContext = &value
return b
}
// WithNodeSelector puts the entries into the NodeSelector field in the declarative configuration
// and returns the receiver, so that objects can be build by chaining "With" function invocations.
// If called multiple times, the entries provided by each call will be put on the NodeSelector field,
// overwriting an existing map entries in NodeSelector field with the same key.
func (b *PodSpecPatchApplyConfiguration) WithNodeSelector(entries map[string]string) *PodSpecPatchApplyConfiguration {
if b.NodeSelector == nil && len(entries) > 0 {
b.NodeSelector = make(map[string]string, len(entries))
}
for k, v := range entries {
b.NodeSelector[k] = v
}
return b
}
// WithAffinity sets the Affinity field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Affinity field is set to the value of the last call.
func (b *PodSpecPatchApplyConfiguration) WithAffinity(value corev1.Affinity) *PodSpecPatchApplyConfiguration {
b.Affinity = &value
return b
}
// WithTolerations adds the given value to the Tolerations field in the declarative configuration
// and returns the receiver, so that objects can be build by chaining "With" function invocations.
// If called multiple times, values provided by each call will be appended to the Tolerations field.
func (b *PodSpecPatchApplyConfiguration) WithTolerations(values ...*v1.TolerationApplyConfiguration) *PodSpecPatchApplyConfiguration {
for i := range values {
if values[i] == nil {
panic("nil value passed to WithTolerations")
}
b.Tolerations = append(b.Tolerations, *values[i])
}
return b
}
// WithSchedulingGates adds the given value to the SchedulingGates field in the declarative configuration
// and returns the receiver, so that objects can be build by chaining "With" function invocations.
// If called multiple times, values provided by each call will be appended to the SchedulingGates field.
func (b *PodSpecPatchApplyConfiguration) WithSchedulingGates(values ...corev1.PodSchedulingGate) *PodSpecPatchApplyConfiguration {
for i := range values {
b.SchedulingGates = append(b.SchedulingGates, values[i])
}
return b
}
pkg/client/applyconfiguration/trainer/v1alpha1/podtemplateoverride.go
Generated
-85
View File
@@ -1,85 +0,0 @@
// Copyright 2024 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by applyconfiguration-gen. DO NOT EDIT.
package v1alpha1
import (
v1 "k8s.io/client-go/applyconfigurations/meta/v1"
)
// PodTemplateOverrideApplyConfiguration represents a declarative configuration of the PodTemplateOverride type for use
// with apply.
//
// PodTemplateOverride represents a custom PodTemplateSpec override that will be applied to the TrainJob's training runtime.
type PodTemplateOverrideApplyConfiguration struct {
// targetJobs is the list of replicated jobs in the training runtime template to apply the overrides.
TargetJobs []PodTemplateOverrideTargetJobApplyConfiguration `json:"targetJobs,omitempty"`
// manager indicates which controller created this PodTemplateOverride object.
// It can be used by external controllers or admission webhooks to track ownership
// and avoid conflicts with other controllers. For example, Kueue sets this field
// to "kueue.k8s.io/manager".
// Defaults to `trainer.kubeflow.org/unknown`
Manager *string `json:"manager,omitempty"`
// metadata overrides the Pod template metadata.
// These values will be merged with the TrainingRuntime's Pod template metadata.
Metadata *v1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"`
// spec overrides the Pod template spec.
// These values will be merged with the TrainingRuntime's Pod template spec.
Spec *PodTemplateSpecOverrideApplyConfiguration `json:"spec,omitempty"`
}
// PodTemplateOverrideApplyConfiguration constructs a declarative configuration of the PodTemplateOverride type for use with
// apply.
func PodTemplateOverride() *PodTemplateOverrideApplyConfiguration {
return &PodTemplateOverrideApplyConfiguration{}
}
// WithTargetJobs adds the given value to the TargetJobs field in the declarative configuration
// and returns the receiver, so that objects can be build by chaining "With" function invocations.
// If called multiple times, values provided by each call will be appended to the TargetJobs field.
func (b *PodTemplateOverrideApplyConfiguration) WithTargetJobs(values ...*PodTemplateOverrideTargetJobApplyConfiguration) *PodTemplateOverrideApplyConfiguration {
for i := range values {
if values[i] == nil {
panic("nil value passed to WithTargetJobs")
}
b.TargetJobs = append(b.TargetJobs, *values[i])
}
return b
}
// WithManager sets the Manager field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Manager field is set to the value of the last call.
func (b *PodTemplateOverrideApplyConfiguration) WithManager(value string) *PodTemplateOverrideApplyConfiguration {
b.Manager = &value
return b
}
// WithMetadata sets the Metadata field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Metadata field is set to the value of the last call.
func (b *PodTemplateOverrideApplyConfiguration) WithMetadata(value *v1.ObjectMetaApplyConfiguration) *PodTemplateOverrideApplyConfiguration {
b.Metadata = value
return b
}
// WithSpec sets the Spec field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Spec field is set to the value of the last call.
func (b *PodTemplateOverrideApplyConfiguration) WithSpec(value *PodTemplateSpecOverrideApplyConfiguration) *PodTemplateOverrideApplyConfiguration {
b.Spec = value
return b
}
pkg/client/applyconfiguration/trainer/v1alpha1/podtemplateoverridetargetjob.go
Generated
-38
View File
@@ -1,38 +0,0 @@
// Copyright 2024 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by applyconfiguration-gen. DO NOT EDIT.
package v1alpha1
// PodTemplateOverrideTargetJobApplyConfiguration represents a declarative configuration of the PodTemplateOverrideTargetJob type for use
// with apply.
type PodTemplateOverrideTargetJobApplyConfiguration struct {
// name is the target training job name for which the PodTemplateSpec is overridden.
Name *string `json:"name,omitempty"`
}
// PodTemplateOverrideTargetJobApplyConfiguration constructs a declarative configuration of the PodTemplateOverrideTargetJob type for use with
// apply.
func PodTemplateOverrideTargetJob() *PodTemplateOverrideTargetJobApplyConfiguration {
return &PodTemplateOverrideTargetJobApplyConfiguration{}
}
// WithName sets the Name field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Name field is set to the value of the last call.
func (b *PodTemplateOverrideTargetJobApplyConfiguration) WithName(value string) *PodTemplateOverrideTargetJobApplyConfiguration {
b.Name = &value
return b
}
pkg/client/applyconfiguration/trainer/v1alpha1/podtemplatepatch.go
Generated
+55
View File
@@ -0,0 +1,55 @@
// Copyright 2024 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by applyconfiguration-gen. DO NOT EDIT.
package v1alpha1
import (
v1 "k8s.io/client-go/applyconfigurations/meta/v1"
)
// PodTemplatePatchApplyConfiguration represents a declarative configuration of the PodTemplatePatch type for use
// with apply.
//
// PodTemplatePatch defines patches for a Pod template within a Job.
type PodTemplatePatchApplyConfiguration struct {
// metadata patches the Pod template metadata.
// Only labels and annotations are allowed.
Metadata *v1.ObjectMetaApplyConfiguration `json:"metadata,omitempty"`
// spec patches the Pod spec with the fields managers are permitted to set.
Spec *PodSpecPatchApplyConfiguration `json:"spec,omitempty"`
}
// PodTemplatePatchApplyConfiguration constructs a declarative configuration of the PodTemplatePatch type for use with
// apply.
func PodTemplatePatch() *PodTemplatePatchApplyConfiguration {
return &PodTemplatePatchApplyConfiguration{}
}
// WithMetadata sets the Metadata field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Metadata field is set to the value of the last call.
func (b *PodTemplatePatchApplyConfiguration) WithMetadata(value *v1.ObjectMetaApplyConfiguration) *PodTemplatePatchApplyConfiguration {
b.Metadata = value
return b
}
// WithSpec sets the Spec field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Spec field is set to the value of the last call.
func (b *PodTemplatePatchApplyConfiguration) WithSpec(value *PodSpecPatchApplyConfiguration) *PodTemplatePatchApplyConfiguration {
b.Spec = value
return b
}
pkg/client/applyconfiguration/trainer/v1alpha1/replicatedjobpatch.go
Generated
+50
View File
@@ -0,0 +1,50 @@
// Copyright 2024 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by applyconfiguration-gen. DO NOT EDIT.
package v1alpha1
// ReplicatedJobPatchApplyConfiguration represents a declarative configuration of the ReplicatedJobPatch type for use
// with apply.
//
// ReplicatedJobPatch defines patches for a specific replicated job within the JobSet.
type ReplicatedJobPatchApplyConfiguration struct {
// name is the name of the replicated job to patch.
Name *string `json:"name,omitempty"`
// template patches the Job template for this replicated job.
Template *JobTemplatePatchApplyConfiguration `json:"template,omitempty"`
}
// ReplicatedJobPatchApplyConfiguration constructs a declarative configuration of the ReplicatedJobPatch type for use with
// apply.
func ReplicatedJobPatch() *ReplicatedJobPatchApplyConfiguration {
return &ReplicatedJobPatchApplyConfiguration{}
}
// WithName sets the Name field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Name field is set to the value of the last call.
func (b *ReplicatedJobPatchApplyConfiguration) WithName(value string) *ReplicatedJobPatchApplyConfiguration {
b.Name = &value
return b
}
// WithTemplate sets the Template field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Template field is set to the value of the last call.
func (b *ReplicatedJobPatchApplyConfiguration) WithTemplate(value *JobTemplatePatchApplyConfiguration) *ReplicatedJobPatchApplyConfiguration {
b.Template = value
return b
}
pkg/client/applyconfiguration/trainer/v1alpha1/runtimepatch.go
Generated
+69
View File
@@ -0,0 +1,69 @@
// Copyright 2024 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by applyconfiguration-gen. DO NOT EDIT.
package v1alpha1
import (
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// RuntimePatchApplyConfiguration represents a declarative configuration of the RuntimePatch type for use
// with apply.
//
// RuntimePatch represents a custom patch applied to the TrainJob's training runtime template.
// Patches are keyed by manager to provide clear ownership and avoid conflicts between controllers.
type RuntimePatchApplyConfiguration struct {
// manager indicates who owns this patch entry. It can be set by the user, external
// controllers, or admission webhooks to track ownership and avoid conflicts.
// For example, Kueue sets this field to "kueue.x-k8s.io/manager".
Manager *string `json:"manager,omitempty"`
// time is the timestamp of when this patch was last written.
// Set by the Trainer admission webhook on each create or update. Used for observability only,
// not used as a list map key.
Time *v1.Time `json:"time,omitempty"`
// trainingRuntimeSpec defines allowed patches for ClusterTrainingRuntime or TrainingRuntime-based jobs.
TrainingRuntimeSpec *TrainingRuntimeSpecPatchApplyConfiguration `json:"trainingRuntimeSpec,omitempty"`
}
// RuntimePatchApplyConfiguration constructs a declarative configuration of the RuntimePatch type for use with
// apply.
func RuntimePatch() *RuntimePatchApplyConfiguration {
return &RuntimePatchApplyConfiguration{}
}
// WithManager sets the Manager field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Manager field is set to the value of the last call.
func (b *RuntimePatchApplyConfiguration) WithManager(value string) *RuntimePatchApplyConfiguration {
b.Manager = &value
return b
}
// WithTime sets the Time field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Time field is set to the value of the last call.
func (b *RuntimePatchApplyConfiguration) WithTime(value v1.Time) *RuntimePatchApplyConfiguration {
b.Time = &value
return b
}
// WithTrainingRuntimeSpec sets the TrainingRuntimeSpec field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the TrainingRuntimeSpec field is set to the value of the last call.
func (b *RuntimePatchApplyConfiguration) WithTrainingRuntimeSpec(value *TrainingRuntimeSpecPatchApplyConfiguration) *RuntimePatchApplyConfiguration {
b.TrainingRuntimeSpec = value
return b
}
pkg/client/applyconfiguration/trainer/v1alpha1/trainingruntimespecpatch.go
Generated
+41
View File
@@ -0,0 +1,41 @@
// Copyright 2024 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Code generated by applyconfiguration-gen. DO NOT EDIT.
package v1alpha1
// TrainingRuntimeSpecPatchApplyConfiguration represents a declarative configuration of the TrainingRuntimeSpecPatch type for use
// with apply.
//
// TrainingRuntimeSpecPatch mirrors TrainingRuntimeSpec but only exposes
// the fields managers are permitted to patch.
type TrainingRuntimeSpecPatchApplyConfiguration struct {
// template patches the JobSet template.
Template *JobSetTemplatePatchApplyConfiguration `json:"template,omitempty"`
}
// TrainingRuntimeSpecPatchApplyConfiguration constructs a declarative configuration of the TrainingRuntimeSpecPatch type for use with
// apply.
func TrainingRuntimeSpecPatch() *TrainingRuntimeSpecPatchApplyConfiguration {
return &TrainingRuntimeSpecPatchApplyConfiguration{}
}
// WithTemplate sets the Template field in the declarative configuration to the given value
// and returns the receiver, so that objects can be built by chaining "With" function invocations.
// If called multiple times, the Template field is set to the value of the last call.
func (b *TrainingRuntimeSpecPatchApplyConfiguration) WithTemplate(value *JobSetTemplatePatchApplyConfiguration) *TrainingRuntimeSpecPatchApplyConfiguration {
b.Template = value
return b
}
pkg/client/applyconfiguration/trainer/v1alpha1/trainjobspec.go
Generated
+8 -42
View File
@@ -27,15 +27,9 @@ type TrainJobSpecApplyConfiguration struct {
Initializer *InitializerApplyConfiguration `json:"initializer,omitempty"`
// trainer defines the configuration of the trainer.
Trainer *TrainerApplyConfiguration `json:"trainer,omitempty"`
// labels to apply for the derivative JobSet and Jobs.
// They will be merged with the TrainingRuntime values.
Labels map[string]string `json:"labels,omitempty"`
// annotations to apply for the derivative JobSet and Jobs.
// They will be merged with the TrainingRuntime values.
Annotations map[string]string `json:"annotations,omitempty"`
// podTemplateOverrides define the PodTemplateOverrides for the training runtime.
// When multiple overrides apply to the same targetJob, later entries in the slice override earlier field values.
PodTemplateOverrides []PodTemplateOverrideApplyConfiguration `json:"podTemplateOverrides,omitempty"`
// runtimePatches defines custom patches applied to the TrainJob's Runtime.
// Patches are keyed by manager to provide clear ownership and avoid conflicts between controllers.
RuntimePatches []RuntimePatchApplyConfiguration `json:"runtimePatches,omitempty"`
// suspend defines whether to suspend the running TrainJob.
Suspend *bool `json:"suspend,omitempty"`
// managedBy is used to indicate the controller or entity that manages a TrainJob.
@@ -77,43 +71,15 @@ func (b *TrainJobSpecApplyConfiguration) WithTrainer(value *TrainerApplyConfigur
return b
}
// WithLabels puts the entries into the Labels field in the declarative configuration
// WithRuntimePatches adds the given value to the RuntimePatches field in the declarative configuration
// and returns the receiver, so that objects can be build by chaining "With" function invocations.
// If called multiple times, the entries provided by each call will be put on the Labels field,
// overwriting an existing map entries in Labels field with the same key.
func (b *TrainJobSpecApplyConfiguration) WithLabels(entries map[string]string) *TrainJobSpecApplyConfiguration {
if b.Labels == nil && len(entries) > 0 {
b.Labels = make(map[string]string, len(entries))
}
for k, v := range entries {
b.Labels[k] = v
}
return b
}
// WithAnnotations puts the entries into the Annotations field in the declarative configuration
// and returns the receiver, so that objects can be build by chaining "With" function invocations.
// If called multiple times, the entries provided by each call will be put on the Annotations field,
// overwriting an existing map entries in Annotations field with the same key.
func (b *TrainJobSpecApplyConfiguration) WithAnnotations(entries map[string]string) *TrainJobSpecApplyConfiguration {
if b.Annotations == nil && len(entries) > 0 {
b.Annotations = make(map[string]string, len(entries))
}
for k, v := range entries {
b.Annotations[k] = v
}
return b
}
// WithPodTemplateOverrides adds the given value to the PodTemplateOverrides field in the declarative configuration
// and returns the receiver, so that objects can be build by chaining "With" function invocations.
// If called multiple times, values provided by each call will be appended to the PodTemplateOverrides field.
func (b *TrainJobSpecApplyConfiguration) WithPodTemplateOverrides(values ...*PodTemplateOverrideApplyConfiguration) *TrainJobSpecApplyConfiguration {
// If called multiple times, values provided by each call will be appended to the RuntimePatches field.
func (b *TrainJobSpecApplyConfiguration) WithRuntimePatches(values ...*RuntimePatchApplyConfiguration) *TrainJobSpecApplyConfiguration {
for i := range values {
if values[i] == nil {
panic("nil value passed to WithPodTemplateOverrides")
panic("nil value passed to WithRuntimePatches")
}
b.PodTemplateOverrides = append(b.PodTemplateOverrides, *values[i])
b.RuntimePatches = append(b.RuntimePatches, *values[i])
}
return b
}
pkg/client/applyconfiguration/utils.go
Generated
+20 -8
View File
@@ -32,8 +32,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} {
// Group=trainer.kubeflow.org, Version=v1alpha1
case v1alpha1.SchemeGroupVersion.WithKind("ClusterTrainingRuntime"):
return &trainerv1alpha1.ClusterTrainingRuntimeApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("ContainerOverride"):
return &trainerv1alpha1.ContainerOverrideApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("ContainerPatch"):
return &trainerv1alpha1.ContainerPatchApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("CoschedulingPodGroupPolicySource"):
return &trainerv1alpha1.CoschedulingPodGroupPolicySourceApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("DatasetInitializer"):
@@ -42,10 +42,18 @@ func ForKind(kind schema.GroupVersionKind) interface{} {
return &trainerv1alpha1.FluxMLPolicySourceApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("Initializer"):
return &trainerv1alpha1.InitializerApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("JobSetSpecPatch"):
return &trainerv1alpha1.JobSetSpecPatchApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("JobSetTemplatePatch"):
return &trainerv1alpha1.JobSetTemplatePatchApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("JobSetTemplateSpec"):
return &trainerv1alpha1.JobSetTemplateSpecApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("JobSpecPatch"):
return &trainerv1alpha1.JobSpecPatchApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("JobStatus"):
return &trainerv1alpha1.JobStatusApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("JobTemplatePatch"):
return &trainerv1alpha1.JobTemplatePatchApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("MLPolicy"):
return &trainerv1alpha1.MLPolicyApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("MLPolicySource"):
@@ -58,12 +66,14 @@ func ForKind(kind schema.GroupVersionKind) interface{} {
return &trainerv1alpha1.PodGroupPolicyApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("PodGroupPolicySource"):
return &trainerv1alpha1.PodGroupPolicySourceApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("PodTemplateOverride"):
return &trainerv1alpha1.PodTemplateOverrideApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("PodTemplateOverrideTargetJob"):
return &trainerv1alpha1.PodTemplateOverrideTargetJobApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("PodTemplateSpecOverride"):
return &trainerv1alpha1.PodTemplateSpecOverrideApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("PodSpecPatch"):
return &trainerv1alpha1.PodSpecPatchApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("PodTemplatePatch"):
return &trainerv1alpha1.PodTemplatePatchApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("ReplicatedJobPatch"):
return &trainerv1alpha1.ReplicatedJobPatchApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("RuntimePatch"):
return &trainerv1alpha1.RuntimePatchApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("RuntimeRef"):
return &trainerv1alpha1.RuntimeRefApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("Trainer"):
@@ -72,6 +82,8 @@ func ForKind(kind schema.GroupVersionKind) interface{} {
return &trainerv1alpha1.TrainingRuntimeApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("TrainingRuntimeSpec"):
return &trainerv1alpha1.TrainingRuntimeSpecApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("TrainingRuntimeSpecPatch"):
return &trainerv1alpha1.TrainingRuntimeSpecPatchApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("TrainJob"):
return &trainerv1alpha1.TrainJobApplyConfiguration{}
case v1alpha1.SchemeGroupVersion.WithKind("TrainJobSpec"):
pkg/controller/clustertrainingruntime_controller_test.go
+2 -2
View File
@@ -150,7 +150,7 @@ func TestNotifyTrainJobUpdate_ClusterTrainingRuntimeReconciler(t *testing.T) {
Obj(),
newJob: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "test").
RuntimeRef(trainer.SchemeGroupVersion.WithKind(trainer.ClusterTrainingRuntimeKind), "test-runtime").
SpecLabel("key", "value").
RuntimePatches([]trainer.RuntimePatch{{Manager: "test"}}).
Obj(),
wantEvent: event.TypedGenericEvent[iter.Seq[types.NamespacedName]]{
Object: func(yield func(types.NamespacedName) bool) {
@@ -164,7 +164,7 @@ func TestNotifyTrainJobUpdate_ClusterTrainingRuntimeReconciler(t *testing.T) {
Obj(),
newJob: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "test").
RuntimeRef(trainer.SchemeGroupVersion.WithKind(trainer.TrainingRuntimeKind), "test-runtime").
SpecLabel("key", "value").
RuntimePatches([]trainer.RuntimePatch{{Manager: "test"}}).
Obj(),
},
"CREATE Event: runtimeRef is ClusterTrainingRuntime": {
pkg/controller/trainingruntime_controller_test.go
+2 -2
View File
@@ -150,7 +150,7 @@ func TestNotifyTrainJobUpdate_TrainingRuntimeReconciler(t *testing.T) {
Obj(),
newJob: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "test").
RuntimeRef(trainer.SchemeGroupVersion.WithKind(trainer.TrainingRuntimeKind), "test-runtime").
SpecLabel("key", "value").
RuntimePatches([]trainer.RuntimePatch{{Manager: "test"}}).
Obj(),
wantEvent: event.TypedGenericEvent[iter.Seq[types.NamespacedName]]{
Object: func(yield func(types.NamespacedName) bool) {
@@ -164,7 +164,7 @@ func TestNotifyTrainJobUpdate_TrainingRuntimeReconciler(t *testing.T) {
Obj(),
newJob: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "test").
RuntimeRef(trainer.SchemeGroupVersion.WithKind(trainer.ClusterTrainingRuntimeKind), "test-runtime").
SpecLabel("key", "value").
RuntimePatches([]trainer.RuntimePatch{{Manager: "test"}}).
Obj(),
},
"CREATE Event: runtimeRef is TrainingRuntime": {
pkg/runtime/core/trainingruntime.go
+47 -57
View File
@@ -21,9 +21,9 @@ import (
"encoding/json"
"errors"
"fmt"
"slices"
"maps"
corev1 "k8s.io/api/core/v1"
batchv1 "k8s.io/api/batch/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
apiruntime "k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
@@ -123,23 +123,25 @@ func (r *TrainingRuntime) RuntimeInfo(
func (r *TrainingRuntime) newRuntimeInfo(
trainJob *trainer.TrainJob, jobSetTemplateSpec trainer.JobSetTemplateSpec, mlPolicy *trainer.MLPolicy, podGroupPolicy *trainer.PodGroupPolicy,
) (*runtime.Info, error) {
propagationLabels := jobSetTemplateSpec.Labels
if propagationLabels == nil && trainJob.Spec.Labels != nil {
propagationLabels = make(map[string]string, len(trainJob.Spec.Labels))
propagationLabels := maps.Clone(jobSetTemplateSpec.Labels)
propagationAnnotations := maps.Clone(jobSetTemplateSpec.Annotations)
for _, patch := range trainJob.Spec.RuntimePatches {
if patch.TrainingRuntimeSpec != nil && patch.TrainingRuntimeSpec.Template != nil && patch.TrainingRuntimeSpec.Template.Metadata != nil {
if propagationLabels == nil && len(patch.TrainingRuntimeSpec.Template.Metadata.Labels) > 0 {
propagationLabels = make(map[string]string)
}
for k, v := range patch.TrainingRuntimeSpec.Template.Metadata.Labels {
propagationLabels[k] = v
}
if propagationAnnotations == nil && len(patch.TrainingRuntimeSpec.Template.Metadata.Annotations) > 0 {
propagationAnnotations = make(map[string]string)
}
for k, v := range patch.TrainingRuntimeSpec.Template.Metadata.Annotations {
propagationAnnotations[k] = v
}
}
}
for k, v := range trainJob.Spec.Labels {
// The JobSetTemplateSpec labels are overridden by the TrainJob Labels (.spec.labels).
propagationLabels[k] = v
}
propagationAnnotations := jobSetTemplateSpec.Annotations
if propagationAnnotations == nil && trainJob.Spec.Annotations != nil {
propagationAnnotations = make(map[string]string, len(trainJob.Spec.Annotations))
}
for k, v := range trainJob.Spec.Annotations {
// The JobSetTemplateSpec annotations are overridden by the TrainJob Annotations (.spec.annotations).
propagationAnnotations[k] = v
}
err := r.mergePodTemplateOverrides(trainJob, &jobSetTemplateSpec)
err := r.mergeRuntimePatches(trainJob, &jobSetTemplateSpec)
if err != nil {
return nil, err
}
@@ -197,51 +199,39 @@ func (r *TrainingRuntime) newRuntimeInfo(
return runtime.NewInfo(opts...), nil
}
func (r *TrainingRuntime) mergePodTemplateOverrides(trainJob *trainer.TrainJob, jobSetTemplateSpec *trainer.JobSetTemplateSpec) error {
for _, podTemplateOverride := range trainJob.Spec.PodTemplateOverrides {
for i, job := range jobSetTemplateSpec.Spec.ReplicatedJobs {
if !slices.ContainsFunc(podTemplateOverride.TargetJobs, func(targetJob trainer.PodTemplateOverrideTargetJob) bool {
return targetJob.Name == job.Name
}) {
func (r *TrainingRuntime) mergeRuntimePatches(trainJob *trainer.TrainJob, jobSetTemplateSpec *trainer.JobSetTemplateSpec) error {
for _, runtimePatch := range trainJob.Spec.RuntimePatches {
if runtimePatch.TrainingRuntimeSpec == nil ||
runtimePatch.TrainingRuntimeSpec.Template == nil ||
runtimePatch.TrainingRuntimeSpec.Template.Spec == nil {
continue
}
for _, rJobPatch := range runtimePatch.TrainingRuntimeSpec.Template.Spec.ReplicatedJobs {
if rJobPatch.Template == nil {
continue
}
podTemplatePatch := map[string]any{}
if podTemplateOverride.Metadata != nil {
metadata := map[string]any{}
if podTemplateOverride.Metadata.Labels != nil {
metadata["labels"] = podTemplateOverride.Metadata.Labels
for i, job := range jobSetTemplateSpec.Spec.ReplicatedJobs {
if job.Name != rJobPatch.Name {
continue
}
if podTemplateOverride.Metadata.Annotations != nil {
metadata["annotations"] = podTemplateOverride.Metadata.Annotations
source, err := json.Marshal(job.Template)
if err != nil {
return err
}
if len(metadata) > 0 {
podTemplatePatch["metadata"] = metadata
patch, err := json.Marshal(rJobPatch.Template)
if err != nil {
return err
}
merged, err := strategicpatch.StrategicMergePatch(source, patch, batchv1.JobTemplateSpec{})
if err != nil {
return err
}
mergedTemplate := batchv1.JobTemplateSpec{}
if err := json.Unmarshal(merged, &mergedTemplate); err != nil {
return err
}
jobSetTemplateSpec.Spec.ReplicatedJobs[i].Template = mergedTemplate
}
if podTemplateOverride.Spec != nil {
podTemplatePatch["spec"] = podTemplateOverride.Spec
}
// Apply a strategic merge patch against the full PodTemplateSpec
source, err := json.Marshal(job.Template.Spec.Template)
if err != nil {
return err
}
patch, err := json.Marshal(podTemplatePatch)
if err != nil {
return err
}
merged, err := strategicpatch.StrategicMergePatch(source, patch, corev1.PodTemplateSpec{})
if err != nil {
return err
}
mergedTemplate := corev1.PodTemplateSpec{}
if err := json.Unmarshal(merged, &mergedTemplate); err != nil {
return err
}
jobSetTemplateSpec.Spec.ReplicatedJobs[i].Template.Spec.Template = mergedTemplate
}
}
return nil
pkg/runtime/core/trainingruntime_test.go
+418 -224
View File
@@ -73,8 +73,17 @@ func TestTrainingRuntimeNewObjects(t *testing.T) {
Suspend(true).
UID("uid").
RuntimeRef(trainer.SchemeGroupVersion.WithKind(trainer.TrainingRuntimeKind), "test-runtime").
SpecLabel("conflictLabel", "override").
SpecAnnotation("conflictAnnotation", "override").
RuntimePatches([]trainer.RuntimePatch{{
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Metadata: &metav1.ObjectMeta{
Labels: map[string]string{"conflictLabel": "override"},
Annotations: map[string]string{"conflictAnnotation": "override"},
},
},
},
}}).
Trainer(
testingutil.MakeTrainJobTrainerWrapper().
NumNodes(30).
@@ -178,7 +187,7 @@ func TestTrainingRuntimeNewObjects(t *testing.T) {
Obj(),
},
},
"succeeded to build JobSet with TrainJob's PodTemplateOverrides.": {
"succeeded to build JobSet with TrainJob's RuntimePatches.": {
trainingRuntime: testingutil.MakeTrainingRuntimeWrapper(metav1.NamespaceDefault, "test-runtime").RuntimeSpec(
testingutil.MakeTrainingRuntimeSpecWrapper(testingutil.MakeTrainingRuntimeWrapper(metav1.NamespaceDefault, "test-runtime").Spec).
WithMLPolicy(
@@ -234,51 +243,100 @@ func TestTrainingRuntimeNewObjects(t *testing.T) {
Container("test:trainjob", []string{"trainjob"}, []string{"trainjob"}, resRequests).
Obj(),
).
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
Manager: ptr.To("manager-1"),
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.DatasetInitializer}},
Spec: &trainer.PodTemplateSpecOverride{
InitContainers: []trainer.ContainerOverride{
{
Name: "override-init-container",
Env: []corev1.EnvVar{
Manager: "manager-1",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{
{
Name: "INIT_ENV",
Value: "override_init",
},
{
Name: "NEW_VALUE",
Value: "from_overrides",
},
},
},
},
Containers: []trainer.ContainerOverride{
{
Name: constants.DatasetInitializer,
VolumeMounts: []corev1.VolumeMount{
{
Name: "initializer_secret",
MountPath: "initializer_secret_mount_path",
},
{
Name: "initializer_claim",
MountPath: "initializer_claim_mount_path",
},
},
},
},
Affinity: &corev1.Affinity{
NodeAffinity: &corev1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
NodeSelectorTerms: []corev1.NodeSelectorTerm{
{
MatchExpressions: []corev1.NodeSelectorRequirement{
{
Key: "topology.kubernetes.io/zone",
Operator: corev1.NodeSelectorOpIn,
Values: []string{"antarctica-east1", "antarctica-west1"},
Name: constants.DatasetInitializer,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
InitContainers: []trainer.ContainerPatch{
{
Name: "override-init-container",
Env: []corev1.EnvVar{
{
Name: "INIT_ENV",
Value: "override_init",
},
{
Name: "NEW_VALUE",
Value: "from_overrides",
},
},
},
},
Containers: []trainer.ContainerPatch{
{
Name: constants.DatasetInitializer,
VolumeMounts: []corev1.VolumeMount{
{
Name: "initializer_secret",
MountPath: "initializer_secret_mount_path",
},
{
Name: "initializer_claim",
MountPath: "initializer_claim_mount_path",
},
},
},
},
Affinity: &corev1.Affinity{
NodeAffinity: &corev1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
NodeSelectorTerms: []corev1.NodeSelectorTerm{
{
MatchExpressions: []corev1.NodeSelectorRequirement{
{
Key: "topology.kubernetes.io/zone",
Operator: corev1.NodeSelectorOpIn,
Values: []string{"antarctica-east1", "antarctica-west1"},
},
},
},
},
},
},
},
NodeSelector: map[string]string{
"node.kubernetes.io/instance-type": "p5.48xlarge",
},
SchedulingGates: []corev1.PodSchedulingGate{
{
Name: "kueue.x-k8s.io/admission",
},
},
Tolerations: []corev1.Toleration{
{
Key: "example.com/gpu",
Operator: corev1.TolerationOpExists,
Effect: corev1.TaintEffectNoSchedule,
},
},
Volumes: []corev1.Volume{
{
Name: "initializer_secret",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: "initializer_secret_name",
},
},
},
{
Name: "initializer_claim",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: "initializer_claim_name",
},
},
},
},
},
},
},
},
@@ -286,91 +344,110 @@ func TestTrainingRuntimeNewObjects(t *testing.T) {
},
},
},
NodeSelector: map[string]string{
"node.kubernetes.io/instance-type": "p5.48xlarge",
},
SchedulingGates: []corev1.PodSchedulingGate{
{
Name: "kueue.x-k8s.io/admission",
},
},
Tolerations: []corev1.Toleration{
{
Key: "example.com/gpu",
Operator: corev1.TolerationOpExists,
Effect: corev1.TaintEffectNoSchedule,
},
},
Volumes: []corev1.Volume{
{
Name: "initializer_secret",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: "initializer_secret_name",
},
},
},
{
Name: "initializer_claim",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: "initializer_claim_name",
},
},
},
},
},
},
{
Manager: ptr.To("manager-2"),
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.Node}},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("override-sa"),
InitContainers: []trainer.ContainerOverride{
{
Name: "override-init-container",
Env: []corev1.EnvVar{
Manager: "manager-2",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{
{
Name: "INIT_ENV",
Value: "override_init",
},
},
},
},
Containers: []trainer.ContainerOverride{
{
Name: constants.Node,
VolumeMounts: []corev1.VolumeMount{
{
Name: "node_secret",
MountPath: "node_secret_mount_path",
},
{
Name: "node_claim",
MountPath: "node_claim_mount_path",
},
},
},
{
Name: "override-container",
Env: []corev1.EnvVar{
{
Name: "CONTAINER_ENV",
Value: "override_container",
},
},
},
},
Affinity: &corev1.Affinity{
NodeAffinity: &corev1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
NodeSelectorTerms: []corev1.NodeSelectorTerm{
{
MatchExpressions: []corev1.NodeSelectorRequirement{
{
Key: "topology.kubernetes.io/zone",
Operator: corev1.NodeSelectorOpIn,
Values: []string{"antarctica-east1", "antarctica-west1"},
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("override-sa"),
InitContainers: []trainer.ContainerPatch{
{
Name: "override-init-container",
Env: []corev1.EnvVar{
{
Name: "INIT_ENV",
Value: "override_init",
},
},
},
},
Containers: []trainer.ContainerPatch{
{
Name: constants.Node,
VolumeMounts: []corev1.VolumeMount{
{
Name: "node_secret",
MountPath: "node_secret_mount_path",
},
{
Name: "node_claim",
MountPath: "node_claim_mount_path",
},
},
},
{
Name: "override-container",
Env: []corev1.EnvVar{
{
Name: "CONTAINER_ENV",
Value: "override_container",
},
},
},
},
Affinity: &corev1.Affinity{
NodeAffinity: &corev1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
NodeSelectorTerms: []corev1.NodeSelectorTerm{
{
MatchExpressions: []corev1.NodeSelectorRequirement{
{
Key: "topology.kubernetes.io/zone",
Operator: corev1.NodeSelectorOpIn,
Values: []string{"antarctica-east1", "antarctica-west1"},
},
},
},
},
},
},
},
ImagePullSecrets: []corev1.LocalObjectReference{
{Name: "registry-credential"},
},
NodeSelector: map[string]string{
"node.kubernetes.io/instance-type": "p5.48xlarge",
},
SchedulingGates: []corev1.PodSchedulingGate{
{
Name: "kueue.x-k8s.io/admission",
},
},
Tolerations: []corev1.Toleration{
{
Key: "example.com/gpu",
Operator: corev1.TolerationOpExists,
Effect: corev1.TaintEffectNoSchedule,
},
},
Volumes: []corev1.Volume{
{
Name: "node_secret",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: "node_secret_name",
},
},
},
{
Name: "node_claim",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: "node_claim_name",
},
},
},
},
},
},
},
},
@@ -378,42 +455,6 @@ func TestTrainingRuntimeNewObjects(t *testing.T) {
},
},
},
ImagePullSecrets: []corev1.LocalObjectReference{
{Name: "registry-credential"},
},
NodeSelector: map[string]string{
"node.kubernetes.io/instance-type": "p5.48xlarge",
},
SchedulingGates: []corev1.PodSchedulingGate{
{
Name: "kueue.x-k8s.io/admission",
},
},
Tolerations: []corev1.Toleration{
{
Key: "example.com/gpu",
Operator: corev1.TolerationOpExists,
Effect: corev1.TaintEffectNoSchedule,
},
},
Volumes: []corev1.Volume{
{
Name: "node_secret",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: "node_secret_name",
},
},
},
{
Name: "node_claim",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: "node_claim_name",
},
},
},
},
},
},
}).
@@ -592,7 +633,7 @@ func TestTrainingRuntimeNewObjects(t *testing.T) {
Obj(),
},
},
"succeeded to build JobSet with nil affinity overrides from the TrainJob's PodTemplateOverrides.": {
"succeeded to build JobSet with nil affinity overrides from the TrainJob's RuntimePatches.": {
trainingRuntime: testingutil.MakeTrainingRuntimeWrapper(metav1.NamespaceDefault, "test-runtime").RuntimeSpec(
testingutil.MakeTrainingRuntimeSpecWrapper(testingutil.MakeTrainingRuntimeWrapper(metav1.NamespaceDefault, "test-runtime").Spec).
WithMLPolicy(
@@ -614,19 +655,51 @@ func TestTrainingRuntimeNewObjects(t *testing.T) {
Container("test:trainjob", []string{"trainjob"}, []string{"trainjob"}, resRequests).
Obj(),
).
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
Manager: ptr.To("manager-1"),
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.DatasetInitializer}},
Spec: &trainer.PodTemplateSpecOverride{
Affinity: nil,
Manager: "manager-1",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{
{
Name: constants.DatasetInitializer,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
Affinity: nil,
},
},
},
},
},
},
},
},
},
},
{
Manager: ptr.To("manager-2"),
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.Node}},
Spec: &trainer.PodTemplateSpecOverride{
Affinity: nil,
Manager: "manager-2",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{
{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
Affinity: nil,
},
},
},
},
},
},
},
},
},
},
}).
@@ -645,7 +718,7 @@ func TestTrainingRuntimeNewObjects(t *testing.T) {
Obj(),
},
},
"succeeded to build JobSet with securityContext overrides from the TrainJob's PodTemplateOverrides.": {
"succeeded to build JobSet with securityContext overrides from the TrainJob's RuntimePatches.": {
trainingRuntime: testingutil.MakeTrainingRuntimeWrapper(metav1.NamespaceDefault, "test-runtime").RuntimeSpec(
testingutil.MakeTrainingRuntimeSpecWrapper(testingutil.MakeTrainingRuntimeWrapper(metav1.NamespaceDefault, "test-runtime").Spec).
WithMLPolicy(
@@ -665,22 +738,39 @@ func TestTrainingRuntimeNewObjects(t *testing.T) {
Container("test:trainjob", []string{"trainjob"}, []string{"trainjob"}, resRequests).
Obj(),
).
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.Node}},
Spec: &trainer.PodTemplateSpecOverride{
SecurityContext: &corev1.PodSecurityContext{
FSGroup: ptr.To(int64(2000)),
RunAsUser: ptr.To(int64(1000)),
RunAsGroup: ptr.To(int64(3000)),
RunAsNonRoot: ptr.To(true),
},
Containers: []trainer.ContainerOverride{
{
Name: "sidecar",
SecurityContext: &corev1.SecurityContext{
AllowPrivilegeEscalation: ptr.To(false),
ReadOnlyRootFilesystem: ptr.To(true),
Manager: "security-manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{
{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
SecurityContext: &corev1.PodSecurityContext{
FSGroup: ptr.To(int64(2000)),
RunAsUser: ptr.To(int64(1000)),
RunAsGroup: ptr.To(int64(3000)),
RunAsNonRoot: ptr.To(true),
},
Containers: []trainer.ContainerPatch{
{
Name: "sidecar",
SecurityContext: &corev1.SecurityContext{
AllowPrivilegeEscalation: ptr.To(false),
ReadOnlyRootFilesystem: ptr.To(true),
},
},
},
},
},
},
},
},
},
},
},
@@ -710,7 +800,7 @@ func TestTrainingRuntimeNewObjects(t *testing.T) {
Obj(),
},
},
"succeeded to build JobSet with labels and annotations overrides from the TrainJob's PodTemplateOverrides.": {
"succeeded to build JobSet with labels and annotations overrides from the TrainJob's RuntimePatches.": {
trainingRuntime: testingutil.MakeTrainingRuntimeWrapper(metav1.NamespaceDefault, "test-runtime").RuntimeSpec(
testingutil.MakeTrainingRuntimeSpecWrapper(testingutil.MakeTrainingRuntimeWrapper(metav1.NamespaceDefault, "test-runtime").Spec).
WithMLPolicy(
@@ -732,21 +822,59 @@ func TestTrainingRuntimeNewObjects(t *testing.T) {
Container("test:trainjob", []string{"trainjob"}, []string{"trainjob"}, resRequests).
Obj(),
).
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
Manager: ptr.To("manager-1"),
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.DatasetInitializer}},
Metadata: &metav1.ObjectMeta{
Labels: map[string]string{"k1": "v1"},
Annotations: map[string]string{"a1": "v1"},
Manager: "manager-1",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{
{
Name: constants.DatasetInitializer,
Template: &trainer.JobTemplatePatch{
Metadata: &metav1.ObjectMeta{
Labels: map[string]string{"job-k1": "job-v1"},
},
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Metadata: &metav1.ObjectMeta{
Labels: map[string]string{"k1": "v1"},
Annotations: map[string]string{"a1": "v1"},
},
},
},
},
},
},
},
},
},
},
{
Manager: ptr.To("manager-2"),
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.Node}},
Metadata: &metav1.ObjectMeta{
Labels: map[string]string{"k2": "v2"},
Annotations: map[string]string{"a2": "v2"},
Manager: "manager-2",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{
{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Metadata: &metav1.ObjectMeta{
Labels: map[string]string{"job-k2": "job-v2"},
},
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Metadata: &metav1.ObjectMeta{
Labels: map[string]string{"k2": "v2"},
Annotations: map[string]string{"a2": "v2"},
},
},
},
},
},
},
},
},
},
},
}).
@@ -762,14 +890,16 @@ func TestTrainingRuntimeNewObjects(t *testing.T) {
InitContainer(constants.Node, "override-init-container", "test:runtime").
Container(constants.Node, constants.Node, "test:trainjob", []string{"trainjob"}, []string{"trainjob"}, resRequests).
Container(constants.Node, "override-container", "test:runtime", []string{"runtime"}, []string{"runtime"}, resRequests).
ReplicatedJobLabel("job-k1", "job-v1", constants.DatasetInitializer).
PodLabelForJobs("k1", "v1", constants.DatasetInitializer).
PodAnnotationForJobs("a1", "v1", constants.DatasetInitializer).
ReplicatedJobLabel("job-k2", "job-v2", constants.Node).
PodLabelForJobs("k2", "v2", constants.Node).
PodAnnotationForJobs("a2", "v2", constants.Node).
Obj(),
},
},
"succeeded to build JobSet with TrainJob's PodTemplateOverrides containing duplicate TargetJobs": {
"succeeded to build JobSet with TrainJob's RuntimePatches containing duplicate patches for same job": {
trainingRuntime: testingutil.MakeTrainingRuntimeWrapper(metav1.NamespaceDefault, "test-runtime").RuntimeSpec(
testingutil.MakeTrainingRuntimeSpecWrapper(testingutil.MakeTrainingRuntimeWrapper(metav1.NamespaceDefault, "test-runtime").Spec).
Container(constants.Node, constants.Node, "test:runtime", []string{"runtime"}, []string{"runtime"}, resRequests).
@@ -778,21 +908,53 @@ func TestTrainingRuntimeNewObjects(t *testing.T) {
trainJob: testingutil.MakeTrainJobWrapper(metav1.NamespaceDefault, "test-job").
UID("uid").
RuntimeRef(trainer.SchemeGroupVersion.WithKind(trainer.TrainingRuntimeKind), "test-runtime").
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
Manager: ptr.To("manager-1"),
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.Node}},
Spec: &trainer.PodTemplateSpecOverride{
NodeSelector: map[string]string{
"node.kubernetes.io/instance-type": "p5.48xlarge",
Manager: "manager-1",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{
{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
NodeSelector: map[string]string{
"node.kubernetes.io/instance-type": "p5.48xlarge",
},
},
},
},
},
},
},
},
},
},
},
{
Manager: ptr.To("manager-1"),
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.Node}},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("test-sa"),
Manager: "manager-1-dup",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{
{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("test-sa"),
},
},
},
},
},
},
},
},
},
},
}).
@@ -812,7 +974,7 @@ func TestTrainingRuntimeNewObjects(t *testing.T) {
Obj(),
},
},
"succeeded to build JobSet with TrainJob's PodTemplateOverrides targeting the same job with different values": {
"succeeded to build JobSet with TrainJob's RuntimePatches targeting the same job with different values": {
trainingRuntime: testingutil.MakeTrainingRuntimeWrapper(metav1.NamespaceDefault, "test-runtime").RuntimeSpec(
testingutil.MakeTrainingRuntimeSpecWrapper(testingutil.MakeTrainingRuntimeWrapper(metav1.NamespaceDefault, "test-runtime").Spec).
Container(constants.Node, constants.Node, "test:runtime", []string{"runtime"}, []string{"runtime"}, resRequests).
@@ -821,22 +983,54 @@ func TestTrainingRuntimeNewObjects(t *testing.T) {
trainJob: testingutil.MakeTrainJobWrapper(metav1.NamespaceDefault, "test-job").
UID("uid").
RuntimeRef(trainer.SchemeGroupVersion.WithKind(trainer.TrainingRuntimeKind), "test-runtime").
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
Manager: ptr.To("manager-1"),
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.Node}},
Spec: &trainer.PodTemplateSpecOverride{
NodeSelector: map[string]string{
"node.kubernetes.io/instance-type": "p5.48xlarge",
Manager: "manager-1",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{
{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
NodeSelector: map[string]string{
"node.kubernetes.io/instance-type": "p5.48xlarge",
},
},
},
},
},
},
},
},
},
},
},
{
Manager: ptr.To("manager-1"),
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.Node}},
Spec: &trainer.PodTemplateSpecOverride{
NodeSelector: map[string]string{
"node.kubernetes.io/instance-type": "p5en.48xlarge",
Manager: "manager-2",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{
{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
NodeSelector: map[string]string{
"node.kubernetes.io/instance-type": "p5en.48xlarge",
},
},
},
},
},
},
},
},
},
},
},
pkg/runtime/framework/plugins/jobset/jobset.go
+36 -29
View File
@@ -51,8 +51,8 @@ import (
)
var (
runtimeRefPath = field.NewPath("spec").Child("runtimeRef")
podTemplateOverridePath = field.NewPath("spec").Child("podTemplateOverrides")
runtimeRefPath = field.NewPath("spec").Child("runtimeRef")
runtimePatchesPath = field.NewPath("spec").Child("runtimePatches")
)
type JobSet struct {
@@ -121,42 +121,49 @@ func (j *JobSet) Validate(ctx context.Context, info *runtime.Info, oldObj, newOb
}
}
allErrs = append(allErrs, j.checkPodTemplateOverridesImmutability(ctx, oldObj, newObj)...)
allErrs = append(allErrs, j.checkRuntimePatchesImmutability(ctx, oldObj, newObj)...)
// TODO (andreyvelich): Validate Volumes, VolumeMounts, and Tolerations.
for _, podTemplateOverride := range newObj.Spec.PodTemplateOverrides {
if podTemplateOverride.Manager != nil {
allErrs = append(allErrs, validation.IsDomainPrefixedPath(podTemplateOverridePath.Child("manager"), *podTemplateOverride.Manager)...)
for _, runtimePatch := range newObj.Spec.RuntimePatches {
allErrs = append(allErrs, validation.IsDomainPrefixedPath(runtimePatchesPath.Child("manager"), runtimePatch.Manager)...)
if runtimePatch.TrainingRuntimeSpec == nil || runtimePatch.TrainingRuntimeSpec.Template == nil ||
runtimePatch.TrainingRuntimeSpec.Template.Spec == nil {
continue
}
for _, targetJob := range podTemplateOverride.TargetJobs {
containers, ok := rJobContainerNames[targetJob.Name]
for _, rJobPatch := range runtimePatch.TrainingRuntimeSpec.Template.Spec.ReplicatedJobs {
containers, ok := rJobContainerNames[rJobPatch.Name]
if !ok {
allErrs = append(allErrs, field.Invalid(podTemplateOverridePath, newObj.Spec.PodTemplateOverrides, "must not have targetJob that doesn't exist in the runtime job template"))
allErrs = append(allErrs, field.Invalid(runtimePatchesPath, newObj.Spec.RuntimePatches,
"must not have replicated job that doesn't exist in the runtime job template"))
continue
}
if podTemplateOverride.Spec != nil {
for _, overrideContainer := range podTemplateOverride.Spec.InitContainers {
if !containers.Has(overrideContainer.Name) {
allErrs = append(allErrs, field.Invalid(podTemplateOverridePath, newObj.Spec.PodTemplateOverrides, fmt.Sprintf("must not have initContainer that doesn't exist in the runtime job %s", targetJob.Name)))
}
if rJobPatch.Template == nil || rJobPatch.Template.Spec == nil ||
rJobPatch.Template.Spec.Template == nil || rJobPatch.Template.Spec.Template.Spec == nil {
continue
}
podSpecPatch := rJobPatch.Template.Spec.Template.Spec
for _, c := range podSpecPatch.InitContainers {
if !containers.Has(c.Name) {
allErrs = append(allErrs, field.Invalid(runtimePatchesPath, newObj.Spec.RuntimePatches,
fmt.Sprintf("must not have initContainer that doesn't exist in the runtime job %s", rJobPatch.Name)))
}
for _, overrideContainer := range podTemplateOverride.Spec.Containers {
if !containers.Has(overrideContainer.Name) {
allErrs = append(allErrs, field.Invalid(podTemplateOverridePath, newObj.Spec.PodTemplateOverrides, fmt.Sprintf("must not have container that doesn't exist in the runtime job %s", targetJob.Name)))
// Trainer and Initializer APIs should be used to set TrainJob envs for the reserved containers.
} else if len(overrideContainer.Env) > 0 && (overrideContainer.Name == constants.DatasetInitializer || overrideContainer.Name == constants.ModelInitializer || overrideContainer.Name == constants.Node) {
allErrs = append(allErrs, field.Invalid(podTemplateOverridePath, newObj.Spec.PodTemplateOverrides,
fmt.Sprintf("must not have envs for the %s, %s, %s containers", constants.DatasetInitializer, constants.ModelInitializer, constants.Node)))
}
}
for _, c := range podSpecPatch.Containers {
if !containers.Has(c.Name) {
allErrs = append(allErrs, field.Invalid(runtimePatchesPath, newObj.Spec.RuntimePatches,
fmt.Sprintf("must not have container that doesn't exist in the runtime job %s", rJobPatch.Name)))
} else if len(c.Env) > 0 && (c.Name == constants.DatasetInitializer || c.Name == constants.ModelInitializer || c.Name == constants.Node) {
allErrs = append(allErrs, field.Invalid(runtimePatchesPath, newObj.Spec.RuntimePatches,
fmt.Sprintf("must not have envs for the %s, %s, %s containers", constants.DatasetInitializer, constants.ModelInitializer, constants.Node)))
}
}
}
}
return nil, allErrs
}
func (j *JobSet) checkPodTemplateOverridesImmutability(ctx context.Context, oldObj, newObj *trainer.TrainJob) field.ErrorList {
func (j *JobSet) checkRuntimePatchesImmutability(ctx context.Context, oldObj, newObj *trainer.TrainJob) field.ErrorList {
var allErrs field.ErrorList
if oldObj == nil {
@@ -165,13 +172,13 @@ func (j *JobSet) checkPodTemplateOverridesImmutability(ctx context.Context, oldO
}
jobSet := &jobsetv1alpha2.JobSet{}
changed := !equality.Semantic.DeepEqual(oldObj.Spec.PodTemplateOverrides, newObj.Spec.PodTemplateOverrides)
changed := !equality.Semantic.DeepEqual(oldObj.Spec.RuntimePatches, newObj.Spec.RuntimePatches)
suspended := ptr.Equal(newObj.Spec.Suspend, ptr.To(true))
if changed {
if !suspended {
allErrs = append(allErrs, field.Forbidden(podTemplateOverridePath, "PodTemplateOverrides can only be modified when the TrainJob is suspended"))
allErrs = append(allErrs, field.Forbidden(runtimePatchesPath, "RuntimePatches can only be modified when the TrainJob is suspended"))
} else if err := j.client.Get(ctx, client.ObjectKeyFromObject(newObj), jobSet); client.IgnoreNotFound(err) != nil {
allErrs = append(allErrs, field.InternalError(podTemplateOverridePath, err))
allErrs = append(allErrs, field.InternalError(runtimePatchesPath, err))
} else {
// If the JobSet exists, check whether it's inactive
// so changes won't have side effects on the JobSet's Pods
@@ -180,8 +187,8 @@ func (j *JobSet) checkPodTemplateOverridesImmutability(ctx context.Context, oldO
// from unsuspended state.
for _, replicatedJob := range jobSet.Status.ReplicatedJobsStatus {
if replicatedJob.Active > 0 {
allErrs = append(allErrs, field.Forbidden(podTemplateOverridePath,
fmt.Sprintf("PodTemplateOverrides cannot be modified when the JobSet's ReplicatedJob %s is still active", replicatedJob.Name)))
allErrs = append(allErrs, field.Forbidden(runtimePatchesPath,
fmt.Sprintf("RuntimePatches cannot be modified when the JobSet's ReplicatedJob %s is still active", replicatedJob.Name)))
}
}
}
pkg/runtime/framework/plugins/jobset/jobset_test.go
+438 -177
View File
@@ -520,7 +520,7 @@ func TestValidate(t *testing.T) {
fmt.Sprintf("must have container with name - %s in the %s job", constants.ModelInitializer, constants.ModelInitializer)),
},
},
"podTemplateOverrides contain invalid targetJob": {
"runtimePatches contain invalid replicated job": {
info: &runtime.Info{
TemplateSpec: runtime.TemplateSpec{
ObjApply: &jobsetv1alpha2ac.JobSetSpecApplyConfiguration{
@@ -542,22 +542,36 @@ func TestValidate(t *testing.T) {
},
},
newObj: utiltesting.MakeTrainJobWrapper("default", "test").
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: "invalid"}},
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{Name: "invalid"}},
},
},
},
},
}).Obj(),
wantError: field.ErrorList{
field.Invalid(podTemplateOverridePath,
[]trainer.PodTemplateOverride{
field.Invalid(runtimePatchesPath,
[]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: "invalid"}},
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{Name: "invalid"}},
},
},
},
},
},
"must not have targetJob that doesn't exist in the runtime job template"),
"must not have replicated job that doesn't exist in the runtime job template"),
},
},
"podTemplateOverrides contain invalid initContainer": {
"runtimePatches contain invalid initContainer": {
info: &runtime.Info{
TemplateSpec: runtime.TemplateSpec{
ObjApply: &jobsetv1alpha2ac.JobSetSpecApplyConfiguration{
@@ -583,27 +597,53 @@ func TestValidate(t *testing.T) {
},
},
newObj: utiltesting.MakeTrainJobWrapper("default", "test").
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.Node}},
Spec: &trainer.PodTemplateSpecOverride{
InitContainers: []trainer.ContainerOverride{
{
Name: "invalid",
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
InitContainers: []trainer.ContainerPatch{
{Name: "invalid"},
},
},
},
},
},
}},
},
},
},
},
}).Obj(),
wantError: field.ErrorList{
field.Invalid(podTemplateOverridePath,
[]trainer.PodTemplateOverride{
field.Invalid(runtimePatchesPath,
[]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.Node}},
Spec: &trainer.PodTemplateSpecOverride{
InitContainers: []trainer.ContainerOverride{
{
Name: "invalid",
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
InitContainers: []trainer.ContainerPatch{
{Name: "invalid"},
},
},
},
},
},
}},
},
},
},
@@ -612,7 +652,7 @@ func TestValidate(t *testing.T) {
fmt.Sprintf("must not have initContainer that doesn't exist in the runtime job %s", constants.Node)),
},
},
"podTemplateOverrides contain invalid container": {
"runtimePatches contain invalid container": {
info: &runtime.Info{
TemplateSpec: runtime.TemplateSpec{
ObjApply: &jobsetv1alpha2ac.JobSetSpecApplyConfiguration{
@@ -638,27 +678,53 @@ func TestValidate(t *testing.T) {
},
},
newObj: utiltesting.MakeTrainJobWrapper("default", "test").
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.Node}},
Spec: &trainer.PodTemplateSpecOverride{
Containers: []trainer.ContainerOverride{
{
Name: "invalid",
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
Containers: []trainer.ContainerPatch{
{Name: "invalid"},
},
},
},
},
},
}},
},
},
},
},
}).Obj(),
wantError: field.ErrorList{
field.Invalid(podTemplateOverridePath,
[]trainer.PodTemplateOverride{
field.Invalid(runtimePatchesPath,
[]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.Node}},
Spec: &trainer.PodTemplateSpecOverride{
Containers: []trainer.ContainerOverride{
{
Name: "invalid",
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
Containers: []trainer.ContainerPatch{
{Name: "invalid"},
},
},
},
},
},
}},
},
},
},
@@ -667,7 +733,7 @@ func TestValidate(t *testing.T) {
fmt.Sprintf("must not have container that doesn't exist in the runtime job %s", constants.Node)),
},
},
"podTemplateOverrides contain envs for reserved container": {
"runtimePatches contain envs for reserved container": {
info: &runtime.Info{
TemplateSpec: runtime.TemplateSpec{
ObjApply: &jobsetv1alpha2ac.JobSetSpecApplyConfiguration{
@@ -693,39 +759,69 @@ func TestValidate(t *testing.T) {
},
},
newObj: utiltesting.MakeTrainJobWrapper("default", "test").
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.Node}},
Spec: &trainer.PodTemplateSpecOverride{
Containers: []trainer.ContainerOverride{
{
Name: constants.Node,
Env: []corev1.EnvVar{
{
Name: "ENV_NAME",
Value: "OVERRIDE",
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
Containers: []trainer.ContainerPatch{
{
Name: constants.Node,
Env: []corev1.EnvVar{
{
Name: "ENV_NAME",
Value: "OVERRIDE",
},
},
},
},
},
},
},
},
},
}},
},
},
},
},
}).Obj(),
wantError: field.ErrorList{
field.Invalid(podTemplateOverridePath,
[]trainer.PodTemplateOverride{
field.Invalid(runtimePatchesPath,
[]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.Node}},
Spec: &trainer.PodTemplateSpecOverride{
Containers: []trainer.ContainerOverride{
{
Name: constants.Node,
Env: []corev1.EnvVar{
{
Name: "ENV_NAME",
Value: "OVERRIDE",
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
Containers: []trainer.ContainerPatch{
{
Name: constants.Node,
Env: []corev1.EnvVar{
{
Name: "ENV_NAME",
Value: "OVERRIDE",
},
},
},
},
},
},
},
},
},
}},
},
},
},
@@ -734,7 +830,7 @@ func TestValidate(t *testing.T) {
fmt.Sprintf("must not have envs for the %s, %s, %s containers", constants.DatasetInitializer, constants.ModelInitializer, constants.Node)),
},
},
"allow podTemplateOverrides when creating a new trainJob": {
"allow runtimePatches when creating a new trainJob": {
info: &runtime.Info{
TemplateSpec: runtime.TemplateSpec{
ObjApply: &jobsetv1alpha2ac.JobSetSpecApplyConfiguration{
@@ -761,22 +857,33 @@ func TestValidate(t *testing.T) {
},
oldObj: nil,
newObj: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "test").
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{
{
Name: constants.Node,
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("service-account"),
},
},
},
},
}},
},
},
},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("service-account"),
},
},
}).
Obj(),
wantError: nil,
},
"allow updates to trainJob with no changes to podTemplateOverrides": {
"allow updates to trainJob with no changes to runtimePatches": {
info: &runtime.Info{
TemplateSpec: runtime.TemplateSpec{
ObjApply: &jobsetv1alpha2ac.JobSetSpecApplyConfiguration{
@@ -802,36 +909,58 @@ func TestValidate(t *testing.T) {
},
},
oldObj: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "test").
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{
{
Name: constants.Node,
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("service-account"),
},
},
},
},
}},
},
},
},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("service-account"),
},
},
}).
Obj(),
newObj: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "test").
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{
{
Name: constants.Node,
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("service-account"),
},
},
},
},
}},
},
},
},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("service-account"),
},
},
}).
Obj(),
wantError: nil,
},
"forbid changes to podTemplateOverrides when trainJob is not suspended": {
"forbid changes to runtimePatches when trainJob is not suspended": {
info: &runtime.Info{
TemplateSpec: runtime.TemplateSpec{
ObjApply: &jobsetv1alpha2ac.JobSetSpecApplyConfiguration{
@@ -858,39 +987,61 @@ func TestValidate(t *testing.T) {
},
oldObj: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "test").
Suspend(false).
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{
{
Name: constants.Node,
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("service-account"),
},
},
},
},
}},
},
},
},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("service-account"),
},
},
}).
Obj(),
newObj: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "test").
Suspend(false).
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{
{
Name: constants.Node,
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("service-account-updated"),
},
},
},
},
}},
},
},
},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("service-account-updated"),
},
},
}).
Obj(),
wantError: field.ErrorList{
field.Forbidden(podTemplateOverridePath, "PodTemplateOverrides can only be modified when the TrainJob is suspended"),
field.Forbidden(runtimePatchesPath, "RuntimePatches can only be modified when the TrainJob is suspended"),
},
},
"allow changes to podTemplateOverrides when trainJob is suspended and jobSet does not exists": {
"allow changes to runtimePatches when trainJob is suspended and jobSet does not exist": {
info: &runtime.Info{
TemplateSpec: runtime.TemplateSpec{
ObjApply: &jobsetv1alpha2ac.JobSetSpecApplyConfiguration{
@@ -917,38 +1068,60 @@ func TestValidate(t *testing.T) {
},
oldObj: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "test").
Suspend(true).
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{
{
Name: constants.Node,
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("service-account"),
},
},
},
},
}},
},
},
},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("service-account"),
},
},
}).
Obj(),
newObj: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "test").
Suspend(true).
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{
{
Name: constants.Node,
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("service-account-updated"),
},
},
},
},
}},
},
},
},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("service-account-updated"),
},
},
}).
Obj(),
clientErr: apierrors.NewNotFound(jobsetv1alpha2.Resource("jobset"), ""),
wantError: nil,
},
"allow changes to podTemplateOverrides when trainJob is suspended and jobSet exists but is inactive": {
"allow changes to runtimePatches when trainJob is suspended and jobSet exists but is inactive": {
info: &runtime.Info{
TemplateSpec: runtime.TemplateSpec{
ObjApply: &jobsetv1alpha2ac.JobSetSpecApplyConfiguration{
@@ -991,31 +1164,53 @@ func TestValidate(t *testing.T) {
},
oldObj: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "test").
Suspend(true).
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{
{
Name: constants.Node,
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("service-account"),
},
},
},
},
}},
},
},
},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("service-account"),
},
},
}).
Obj(),
newObj: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "test").
Suspend(true).
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{
{
Name: constants.Node,
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("service-account-updated"),
},
},
},
},
}},
},
},
},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("service-account-updated"),
},
},
}).
Obj(),
@@ -1039,7 +1234,7 @@ func TestValidate(t *testing.T) {
},
wantError: nil,
},
"forbid changes to podTemplateOverrides when trainJob is suspended but jobSet has an active replicatedJob": {
"forbid changes to runtimePatches when trainJob is suspended but jobSet has an active replicatedJob": {
info: &runtime.Info{
TemplateSpec: runtime.TemplateSpec{
ObjApply: &jobsetv1alpha2ac.JobSetSpecApplyConfiguration{
@@ -1082,31 +1277,53 @@ func TestValidate(t *testing.T) {
},
oldObj: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "test").
Suspend(true).
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{
{
Name: constants.Node,
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("service-account"),
},
},
},
},
}},
},
},
},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("service-account"),
},
},
}).
Obj(),
newObj: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "test").
Suspend(true).
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{
{
Name: constants.Node,
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("service-account-updated"),
},
},
},
},
}},
},
},
},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("service-account-updated"),
},
},
}).
Obj(),
@@ -1129,10 +1346,10 @@ func TestValidate(t *testing.T) {
},
},
wantError: field.ErrorList{
field.Forbidden(podTemplateOverridePath, "PodTemplateOverrides cannot be modified when the JobSet's ReplicatedJob node is still active"),
field.Forbidden(runtimePatchesPath, "RuntimePatches cannot be modified when the JobSet's ReplicatedJob node is still active"),
},
},
"forbid changes to podTemplateOverrides when trainJob is suspended but has multiple active replicatedJobs": {
"forbid changes to runtimePatches when trainJob is suspended but has multiple active replicatedJobs": {
info: &runtime.Info{
TemplateSpec: runtime.TemplateSpec{
ObjApply: &jobsetv1alpha2ac.JobSetSpecApplyConfiguration{
@@ -1175,31 +1392,53 @@ func TestValidate(t *testing.T) {
},
oldObj: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "test").
Suspend(true).
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{
{
Name: constants.Node,
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("service-account"),
},
},
},
},
}},
},
},
},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("service-account"),
},
},
}).
Obj(),
newObj: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "test").
Suspend(true).
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{
{
Name: constants.Node,
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("service-account-updated"),
},
},
},
},
}},
},
},
},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("service-account-updated"),
},
},
}).
Obj(),
@@ -1222,11 +1461,11 @@ func TestValidate(t *testing.T) {
},
},
wantError: field.ErrorList{
field.Forbidden(podTemplateOverridePath, "PodTemplateOverrides cannot be modified when the JobSet's ReplicatedJob dataset-initializer is still active"),
field.Forbidden(podTemplateOverridePath, "PodTemplateOverrides cannot be modified when the JobSet's ReplicatedJob node is still active"),
field.Forbidden(runtimePatchesPath, "RuntimePatches cannot be modified when the JobSet's ReplicatedJob dataset-initializer is still active"),
field.Forbidden(runtimePatchesPath, "RuntimePatches cannot be modified when the JobSet's ReplicatedJob node is still active"),
},
},
"forbid changes to podTemplateOverrides when trainJob is suspended but jobSet cannot be checked due to a client error": {
"forbid changes to runtimePatches when trainJob is suspended but jobSet cannot be checked due to a client error": {
info: &runtime.Info{
TemplateSpec: runtime.TemplateSpec{
ObjApply: &jobsetv1alpha2ac.JobSetSpecApplyConfiguration{
@@ -1253,37 +1492,59 @@ func TestValidate(t *testing.T) {
},
oldObj: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "test").
Suspend(true).
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{
{
Name: constants.Node,
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("service-account"),
},
},
},
},
}},
},
},
},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("service-account"),
},
},
}).
Obj(),
newObj: utiltesting.MakeTrainJobWrapper(metav1.NamespaceDefault, "test").
Suspend(true).
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{
{
Name: constants.Node,
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("service-account-updated"),
},
},
},
},
}},
},
},
},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("service-account-updated"),
},
},
}).
Obj(),
clientErr: fmt.Errorf("client error"),
wantError: field.ErrorList{
field.InternalError(podTemplateOverridePath, fmt.Errorf("client error")),
field.InternalError(runtimePatchesPath, fmt.Errorf("client error")),
},
},
}
pkg/util/testing/wrapper.go
+2 -18
View File
@@ -611,22 +611,6 @@ func (t *TrainJobWrapper) UID(uid string) *TrainJobWrapper {
return t
}
func (t *TrainJobWrapper) SpecLabel(key, value string) *TrainJobWrapper {
if t.Spec.Labels == nil {
t.Spec.Labels = make(map[string]string, 1)
}
t.Spec.Labels[key] = value
return t
}
func (t *TrainJobWrapper) SpecAnnotation(key, value string) *TrainJobWrapper {
if t.Spec.Annotations == nil {
t.Spec.Annotations = make(map[string]string, 1)
}
t.Spec.Annotations[key] = value
return t
}
func (t *TrainJobWrapper) RuntimeRef(gvk schema.GroupVersionKind, name string) *TrainJobWrapper {
runtimeRef := trainer.RuntimeRef{
Name: name,
@@ -651,8 +635,8 @@ func (t *TrainJobWrapper) Trainer(trainer *trainer.Trainer) *TrainJobWrapper {
return t
}
func (t *TrainJobWrapper) PodTemplateOverrides(podTemplateOverrides []trainer.PodTemplateOverride) *TrainJobWrapper {
t.Spec.PodTemplateOverrides = podTemplateOverrides
func (t *TrainJobWrapper) RuntimePatches(patches []trainer.RuntimePatch) *TrainJobWrapper {
t.Spec.RuntimePatches = patches
return t
}
test/e2e/e2e_test.go
+44 -15
View File
@@ -298,38 +298,67 @@ var _ = ginkgo.Describe("TrainJob e2e", func() {
})
})
ginkgo.When("Creating a TrainJob with PodTemplateOverrides", func() {
ginkgo.It("should allow the user to set the manager field and default to 'Unknown' if empty", func() {
ginkgo.When("Creating a TrainJob with RuntimePatches", func() {
ginkgo.It("should preserve user-provided manager fields", func() {
trainJob := testingutil.MakeTrainJobWrapper(ns.Name, "e2e-test").
RuntimeRef(trainer.SchemeGroupVersion.WithKind(trainer.ClusterTrainingRuntimeKind), torchRuntime).
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.Node}},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("test-sa-1"),
Manager: "test.io/manager-one",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("test-sa-1"),
},
},
},
},
}},
},
},
},
},
{
Manager: ptr.To("kueue.k8s.io/manager"),
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.Node}},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("test-sa-2"),
Manager: "kueue.k8s.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("test-sa-2"),
},
},
},
},
}},
},
},
},
},
}).
Obj()
ginkgo.By("Create a TrainJob with PodTemplateOverrides", func() {
ginkgo.By("Create a TrainJob with RuntimePatches", func() {
gomega.Expect(k8sClient.Create(ctx, trainJob)).Should(gomega.Succeed())
})
ginkgo.By("Verify manager field preserves user input and defaults to 'Unknown' if empty", func() {
ginkgo.By("Verify manager fields are preserved", func() {
gomega.Eventually(func(g gomega.Gomega) {
gotTrainJob := &trainer.TrainJob{}
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(trainJob), gotTrainJob)).Should(gomega.Succeed())
g.Expect(gotTrainJob.Spec.PodTemplateOverrides).Should(gomega.HaveLen(2))
g.Expect(*gotTrainJob.Spec.PodTemplateOverrides[0].Manager).To(gomega.Equal("trainer.kubeflow.org/unknown"))
g.Expect(*gotTrainJob.Spec.PodTemplateOverrides[1].Manager).To(gomega.Equal("kueue.k8s.io/manager"))
g.Expect(gotTrainJob.Spec.RuntimePatches).Should(gomega.HaveLen(2))
g.Expect(gotTrainJob.Spec.RuntimePatches[0].Manager).To(gomega.Equal("test.io/manager-one"))
g.Expect(gotTrainJob.Spec.RuntimePatches[1].Manager).To(gomega.Equal("kueue.k8s.io/manager"))
}, util.Timeout, util.Interval).Should(gomega.Succeed())
})
})
test/integration/controller/trainjob_controller_test.go
+76 -39
View File
@@ -85,8 +85,17 @@ var _ = ginkgo.Describe("TrainJob controller", ginkgo.Ordered, func() {
trainJob = testingutil.MakeTrainJobWrapper(ns.Name, "alpha").
Suspend(true).
RuntimeRef(trainer.GroupVersion.WithKind(trainer.TrainingRuntimeKind), "alpha").
SpecLabel("testingKey", "testingVal").
SpecAnnotation("testingKey", "testingVal").
RuntimePatches([]trainer.RuntimePatch{{
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Metadata: &metav1.ObjectMeta{
Labels: map[string]string{"testingKey": "testingVal"},
Annotations: map[string]string{"testingKey": "testingVal"},
},
},
},
}}).
Trainer(
testingutil.MakeTrainJobTrainerWrapper().
Container("test:trainjob", []string{"trainjob"}, []string{"trainjob"}, resRequests).
@@ -201,22 +210,35 @@ var _ = ginkgo.Describe("TrainJob controller", ginkgo.Ordered, func() {
ginkgo.By("Updating suspended TrainJob node selector")
updatedSelector := map[string]string{"updated": "selector"}
podTemplateOverrides := []trainer.PodTemplateOverride{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{
{
Name: "node",
runtimePatches := []trainer.RuntimePatch{{
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Metadata: &metav1.ObjectMeta{
Labels: map[string]string{"testingKey": "testingVal"},
Annotations: map[string]string{"testingKey": "testingVal"},
},
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: "node",
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
NodeSelector: updatedSelector,
},
},
},
},
}},
},
},
Spec: &trainer.PodTemplateSpecOverride{
NodeSelector: updatedSelector,
},
},
}
}}
gomega.Eventually(func(g gomega.Gomega) {
g.Expect(k8sClient.Get(ctx, trainJobKey, trainJob)).Should(gomega.Succeed())
trainJob.Spec.PodTemplateOverrides = podTemplateOverrides
trainJob.Spec.RuntimePatches = runtimePatches
g.Expect(k8sClient.Update(ctx, trainJob)).Should(gomega.Succeed())
}, util.Timeout, util.Interval).Should(gomega.Succeed())
@@ -773,7 +795,7 @@ var _ = ginkgo.Describe("TrainJob controller", ginkgo.Ordered, func() {
}, util.Timeout, util.Interval).Should(gomega.Succeed())
})
ginkgo.It("Should succeed to create TrainJob with PodTemplateOverrides", func() {
ginkgo.It("Should succeed to create TrainJob with RuntimePatches", func() {
ginkgo.By("Creating Torch TrainingRuntime and TrainJob")
trainJob = testingutil.MakeTrainJobWrapper(ns.Name, "alpha").
RuntimeRef(trainer.GroupVersion.WithKind(trainer.TrainingRuntimeKind), "alpha").
@@ -782,34 +804,49 @@ var _ = ginkgo.Describe("TrainJob controller", ginkgo.Ordered, func() {
Container("test:trainjob", []string{"trainjob"}, []string{"trainjob"}, resRequests).
Env([]corev1.EnvVar{{Name: "TRAIN_JOB", Value: "value"}}...).
Obj()).
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: constants.Node}},
Metadata: &metav1.ObjectMeta{
Labels: map[string]string{
"override-label-key": "override-label-value",
"custom-label": "custom-value",
},
Annotations: map[string]string{
"override-annotation-key": "override-annotation-value",
"custom-annotation": "custom-annotation-value",
},
},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("override-sa"),
InitContainers: []trainer.ContainerOverride{
{
Name: "override-init-container",
Env: []corev1.EnvVar{
{
Name: "INIT_ENV",
Value: "override_init",
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: constants.Node,
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Metadata: &metav1.ObjectMeta{
Labels: map[string]string{
"override-label-key": "override-label-value",
"custom-label": "custom-value",
},
Annotations: map[string]string{
"override-annotation-key": "override-annotation-value",
"custom-annotation": "custom-annotation-value",
},
},
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("override-sa"),
InitContainers: []trainer.ContainerPatch{
{
Name: "override-init-container",
Env: []corev1.EnvVar{
{
Name: "INIT_ENV",
Value: "override_init",
},
{
Name: "NEW_VALUE",
Value: "from_overrides",
},
},
},
},
},
},
},
},
{
Name: "NEW_VALUE",
Value: "from_overrides",
},
},
}},
},
},
},
test/integration/webhooks/trainjob_test.go
+69 -57
View File
@@ -330,48 +330,76 @@ var _ = ginkgo.Describe("TrainJob marker validations and defaulting", ginkgo.Ord
Obj()
},
testingutil.BeInvalidError()),
ginkgo.Entry("Should succeed to create trainJob with podTemplateOverrides containing duplicate targetJob",
ginkgo.Entry("Should succeed to create trainJob with runtimePatches containing two replicated job patches",
func() *trainer.TrainJob {
return testingutil.MakeTrainJobWrapper(ns.Name, "duplicated-podspecoverrides-target-jobs").
return testingutil.MakeTrainJobWrapper(ns.Name, "two-rjob-patches").
RuntimeRef(trainer.GroupVersion.WithKind(trainer.TrainingRuntimeKind), "testing").
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: "node"}},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("custom-sa"),
Manager: "acme.io/manager-one",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: "node",
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("custom-sa"),
},
},
},
},
}},
},
},
},
},
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: "node"}},
Spec: &trainer.PodTemplateSpecOverride{
ServiceAccountName: ptr.To("custom-sa-two"),
Manager: "acme.io/manager-two",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: "node",
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
ServiceAccountName: ptr.To("custom-sa-two"),
},
},
},
},
}},
},
},
},
},
}).
Obj()
},
gomega.Succeed()),
ginkgo.Entry("Should succeed to create trainJob with valid PodTemplateOverrides.Manager",
ginkgo.Entry("Should succeed to create trainJob with valid RuntimePatches manager",
func() *trainer.TrainJob {
return testingutil.MakeTrainJobWrapper(ns.Name, "valid-podtemplateoverrides-manager").
return testingutil.MakeTrainJobWrapper(ns.Name, "valid-runtimepatches-manager").
RuntimeRef(trainer.GroupVersion.WithKind(trainer.TrainingRuntimeKind), "testing").
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: "node"}},
Manager: ptr.To("kueue.k8s.io/manager"),
Manager: "kueue.k8s.io/manager",
},
}).
Obj()
},
gomega.Succeed()),
ginkgo.Entry("Should fail to create trainJob with invalid PodTemplateOverrides.Manager",
ginkgo.Entry("Should fail to create trainJob with invalid RuntimePatches manager",
func() *trainer.TrainJob {
return testingutil.MakeTrainJobWrapper(ns.Name, "invalid-podtemplateoverrides-manager").
return testingutil.MakeTrainJobWrapper(ns.Name, "invalid-runtimepatches-manager").
RuntimeRef(trainer.GroupVersion.WithKind(trainer.TrainingRuntimeKind), "testing").
PodTemplateOverrides([]trainer.PodTemplateOverride{
RuntimePatches([]trainer.RuntimePatch{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: "node"}},
Manager: ptr.To("kueue.k8s.io/"),
Manager: "kueue.k8s.io/",
},
}).
Obj()
@@ -397,31 +425,6 @@ var _ = ginkgo.Describe("TrainJob marker validations and defaulting", ginkgo.Ord
Suspend(false).
Obj()
}),
ginkgo.Entry("Should succeed to default PodTemplateOverrides.Manager='trainer.kubeflow.org/unknown'",
func() *trainer.TrainJob {
return testingutil.MakeTrainJobWrapper(ns.Name, "null-podtemplateoverrides-manager").
RuntimeRef(trainer.SchemeGroupVersion.WithKind(trainer.TrainingRuntimeKind), "testing").
PodTemplateOverrides([]trainer.PodTemplateOverride{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: "node"}},
},
}).
Suspend(true).
Obj()
},
func() *trainer.TrainJob {
return testingutil.MakeTrainJobWrapper(ns.Name, "null-podtemplateoverrides-manager").
ManagedBy("trainer.kubeflow.org/trainjob-controller").
RuntimeRef(trainer.SchemeGroupVersion.WithKind(trainer.TrainingRuntimeKind), "testing").
PodTemplateOverrides([]trainer.PodTemplateOverride{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{{Name: "node"}},
Manager: ptr.To("trainer.kubeflow.org/unknown"),
},
}).
Suspend(true).
Obj()
}),
ginkgo.Entry("Should succeed to default managedBy=trainer.kubeflow.org/trainjob-controller",
func() *trainer.TrainJob {
return testingutil.MakeTrainJobWrapper(ns.Name, "null-managed-by").
@@ -527,29 +530,38 @@ var _ = ginkgo.Describe("TrainJob marker validations and defaulting", ginkgo.Ord
return job
},
testingutil.BeInvalidError()),
ginkgo.Entry("Should succeed to update podTemplateOverride when suspend is true",
ginkgo.Entry("Should succeed to update runtimePatches when suspend is true",
func() *trainer.TrainJob {
return testingutil.MakeTrainJobWrapper(ns.Name, "valid-trainer").
return testingutil.MakeTrainJobWrapper(ns.Name, "valid-runtimepatches").
RuntimeRef(trainer.SchemeGroupVersion.WithKind(trainer.TrainingRuntimeKind), "testing").
Suspend(true).
PodTemplateOverrides(
[]trainer.PodTemplateOverride{
{
TargetJobs: []trainer.PodTemplateOverrideTargetJob{
{
Name: "node",
RuntimePatches([]trainer.RuntimePatch{
{
Manager: "test.io/manager",
TrainingRuntimeSpec: &trainer.TrainingRuntimeSpecPatch{
Template: &trainer.JobSetTemplatePatch{
Spec: &trainer.JobSetSpecPatch{
ReplicatedJobs: []trainer.ReplicatedJobPatch{{
Name: "node",
Template: &trainer.JobTemplatePatch{
Spec: &trainer.JobSpecPatch{
Template: &trainer.PodTemplatePatch{
Spec: &trainer.PodSpecPatch{
NodeSelector: map[string]string{"test": "test"},
},
},
},
},
}},
},
},
Spec: &trainer.PodTemplateSpecOverride{
NodeSelector: map[string]string{"test": "test"},
},
},
},
).
}).
Obj()
},
func(job *trainer.TrainJob) *trainer.TrainJob {
job.Spec.PodTemplateOverrides[0].Spec.NodeSelector = map[string]string{"allow": "update"}
job.Spec.RuntimePatches[0].TrainingRuntimeSpec.Template.Spec.ReplicatedJobs[0].Template.Spec.Template.Spec.NodeSelector = map[string]string{"allow": "update"}
return job
},
gomega.Succeed()),