feat(runtimes): Add label to disable webhook validation for Runtimes (#3683)

* feat(runtimes): Add label to disable webhook validation for Runtimes Signed-off-by: Andrey Velichkevich <[email protected]> * Use commonLabels to set labels Signed-off-by: Andrey Velichkevich <[email protected]> * Set resources to base runtimes in E2Es Signed-off-by: Andrey Velichkevich <[email protected]> * Fix Helm runtime images in e2e Signed-off-by: Andrey Velichkevich <[email protected]> --------- Signed-off-by: Andrey Velichkevich <[email protected]>
2026-09-24 00:31:34 +00:00 · 2026-07-03 13:37:21 +00:00
parent 49f4eaadd2
commit 663ae38567
27 changed files with 171 additions and 17 deletions
@@ -151,8 +151,9 @@ manager:
 | dataCache.cacheImage.tag | string | `""` | Data cache image tag. Defaults to chart version if empty. |
 | dataCache.runtimes.torchDistributedWithCache | object | `{"enabled":false}` | PyTorch distributed training with data cache support |
 | dataCache.runtimes.torchDistributedWithCache.enabled | bool | `false` | Enable deployment of torch-distributed-with-cache runtime |
-| runtimes | object | `{"deepspeedDistributed":{"enabled":false,"image":{"registry":"ghcr.io","repository":"kubeflow/trainer/deepspeed-runtime","tag":""}},"defaultEnabled":false,"jaxDistributed":{"enabled":false},"mlxDistributed":{"enabled":false,"image":{"registry":"ghcr.io","repository":"kubeflow/trainer/mlx-runtime","tag":""}},"torchDistributed":{"enabled":false},"torchtuneDistributed":{"image":{"registry":"ghcr.io","repository":"kubeflow/trainer/torchtune-trainer","tag":""},"llama3_2_1B":{"enabled":false},"llama3_2_3B":{"enabled":false},"qwen2_5_1_5B":{"enabled":false}},"xgboostDistributed":{"enabled":false,"image":{"registry":"ghcr.io","repository":"kubeflow/trainer/xgboost-runtime","tag":""}}}` | ClusterTrainingRuntimes configuration These are optional runtime templates that can be deployed with the Helm chart. Each runtime provides a blueprint for different ML frameworks and configurations. |
+| runtimes | object | `{"commonLabels":{"trainer.kubeflow.org/webhook-validation":"disabled"},"deepspeedDistributed":{"enabled":false,"image":{"registry":"ghcr.io","repository":"kubeflow/trainer/deepspeed-runtime","tag":""}},"defaultEnabled":false,"jaxDistributed":{"enabled":false},"mlxDistributed":{"enabled":false,"image":{"registry":"ghcr.io","repository":"kubeflow/trainer/mlx-runtime","tag":""}},"torchDistributed":{"enabled":false},"torchtuneDistributed":{"image":{"registry":"ghcr.io","repository":"kubeflow/trainer/torchtune-trainer","tag":""},"llama3_2_1B":{"enabled":false},"llama3_2_3B":{"enabled":false},"qwen2_5_1_5B":{"enabled":false}},"xgboostDistributed":{"enabled":false,"image":{"registry":"ghcr.io","repository":"kubeflow/trainer/xgboost-runtime","tag":""}}}` | ClusterTrainingRuntimes configuration These are optional runtime templates that can be deployed with the Helm chart. Each runtime provides a blueprint for different ML frameworks and configurations. |
 | runtimes.defaultEnabled | bool | `false` | Enable all default runtimes (torch, deepspeed, mlx, jax, torchtune) when set to true. Individual runtime settings will be ignored if this is enabled. |
+| runtimes.commonLabels | object | `{"trainer.kubeflow.org/webhook-validation":"disabled"}` | Common labels applied to every built-in runtime. The built-in ClusterTrainingRuntime are validated ahead of time. |
 | runtimes.torchDistributed | object | `{"enabled":false}` | PyTorch distributed training runtime (no custom images required) |
 | runtimes.torchDistributed.enabled | bool | `false` | Enable deployment of torch-distributed runtime |
 | runtimes.deepspeedDistributed | object | `{"enabled":false,"image":{"registry":"ghcr.io","repository":"kubeflow/trainer/deepspeed-runtime","tag":""}}` | DeepSpeed distributed training runtime |
@@ -25,6 +25,9 @@ metadata:
  name: torch-distributed-with-cache
  labels:
    trainer.kubeflow.org/framework: torch
+    {{- with .Values.runtimes.commonLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
    {{- include "trainer.labels" . | nindent 4 }}
 spec:
  mlPolicy:
@@ -21,6 +21,9 @@ metadata:
  name: deepspeed-distributed
  labels:
    trainer.kubeflow.org/framework: deepspeed
+    {{- with .Values.runtimes.commonLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
    {{- include "trainer.labels" . | nindent 4 }}
 spec:
  mlPolicy:
@@ -21,6 +21,9 @@ metadata:
  name: jax-distributed
  labels:
    trainer.kubeflow.org/framework: jax
+    {{- with .Values.runtimes.commonLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
    {{- include "trainer.labels" . | nindent 4 }}
 spec:
  mlPolicy:
@@ -21,6 +21,9 @@ metadata:
  name: mlx-distributed
  labels:
    trainer.kubeflow.org/framework: mlx
+    {{- with .Values.runtimes.commonLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
    {{- include "trainer.labels" . | nindent 4 }}
 spec:
  mlPolicy:
@@ -21,6 +21,9 @@ metadata:
  name: torch-distributed
  labels:
    trainer.kubeflow.org/framework: torch
+    {{- with .Values.runtimes.commonLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
    {{- include "trainer.labels" . | nindent 4 }}
 spec:
  mlPolicy:
@@ -21,6 +21,9 @@ metadata:
  name: torchtune-llama3.2-1b
  labels:
    trainer.kubeflow.org/framework: torchtune
+    {{- with .Values.runtimes.commonLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
    {{- include "trainer.labels" . | nindent 4 }}
 spec:
  mlPolicy:
@@ -21,6 +21,9 @@ metadata:
  name: torchtune-llama3.2-3b
  labels:
    trainer.kubeflow.org/framework: torchtune
+    {{- with .Values.runtimes.commonLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
    {{- include "trainer.labels" . | nindent 4 }}
 spec:
  mlPolicy:
@@ -21,6 +21,9 @@ metadata:
  name: torchtune-qwen2.5-1.5b
  labels:
    trainer.kubeflow.org/framework: torchtune
+    {{- with .Values.runtimes.commonLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
    {{- include "trainer.labels" . | nindent 4 }}
 spec:
  mlPolicy:
@@ -21,6 +21,9 @@ metadata:
  name: xgboost-distributed
  labels:
    trainer.kubeflow.org/framework: xgboost
+    {{- with .Values.runtimes.commonLabels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
    {{- include "trainer.labels" . | nindent 4 }}
 spec:
  mlPolicy:
@@ -33,6 +33,12 @@ webhooks:
  {{- with .Values.webhook.failurePolicy }}
  failurePolicy: {{ . }}
  {{- end }}
+  objectSelector:
+    matchExpressions:
+    - key: trainer.kubeflow.org/webhook-validation
+      operator: NotIn
+      values:
+      - disabled
  rules:
  - apiGroups:
    - trainer.kubeflow.org
@@ -55,6 +61,12 @@ webhooks:
  {{- with .Values.webhook.failurePolicy }}
  failurePolicy: {{ . }}
  {{- end }}
+  objectSelector:
+    matchExpressions:
+    - key: trainer.kubeflow.org/webhook-validation
+      operator: NotIn
+      values:
+      - disabled
  rules:
  - apiGroups:
    - trainer.kubeflow.org
@@ -65,6 +65,9 @@ tests:
      - equal:
          path: metadata.labels["trainer.kubeflow.org/framework"]
          value: deepspeed
+      - equal:
+          path: metadata.labels["trainer.kubeflow.org/webhook-validation"]
+          value: disabled

  - it: Should have correct MPI mlPolicy configuration
    set:
@@ -65,6 +65,9 @@ tests:
      - equal:
          path: metadata.labels["trainer.kubeflow.org/framework"]
          value: mlx
+      - equal:
+          path: metadata.labels["trainer.kubeflow.org/webhook-validation"]
+          value: disabled

  - it: Should have correct MPI mlPolicy configuration
    set:
@@ -53,6 +53,9 @@ tests:
      - equal:
          path: metadata.labels["trainer.kubeflow.org/framework"]
          value: torch
+      - equal:
+          path: metadata.labels["trainer.kubeflow.org/webhook-validation"]
+          value: disabled

  - it: Should have correct mlPolicy configuration
    set:
@@ -86,6 +86,9 @@ tests:
      - equal:
          path: metadata.labels["trainer.kubeflow.org/framework"]
          value: torch
+      - equal:
+          path: metadata.labels["trainer.kubeflow.org/webhook-validation"]
+          value: disabled

  - it: Should use correct serviceAccountName for dataset-initializer
    set:
@@ -69,6 +69,9 @@ tests:
      - equal:
          path: metadata.labels["trainer.kubeflow.org/framework"]
          value: torchtune
+      - equal:
+          path: metadata.labels["trainer.kubeflow.org/webhook-validation"]
+          value: disabled

  - it: Should have correct mlPolicy configuration
    set:
@@ -69,6 +69,9 @@ tests:
      - equal:
          path: metadata.labels["trainer.kubeflow.org/framework"]
          value: torchtune
+      - equal:
+          path: metadata.labels["trainer.kubeflow.org/webhook-validation"]
+          value: disabled

  - it: Should have correct mlPolicy configuration
    set:
@@ -69,6 +69,9 @@ tests:
      - equal:
          path: metadata.labels["trainer.kubeflow.org/framework"]
          value: torchtune
+      - equal:
+          path: metadata.labels["trainer.kubeflow.org/webhook-validation"]
+          value: disabled

  - it: Should have correct mlPolicy configuration
    set:
@@ -65,6 +65,9 @@ tests:
      - equal:
          path: metadata.labels["trainer.kubeflow.org/framework"]
          value: xgboost
+      - equal:
+          path: metadata.labels["trainer.kubeflow.org/webhook-validation"]
+          value: disabled

  - it: Should have correct XGBoost mlPolicy configuration
    set:
@@ -39,3 +39,26 @@ tests:
      - equal:
          path: webhooks[*].failurePolicy
          value: Ignore
+
+  - it: Should skip validation for runtimes labeled trainer.kubeflow.org/webhook-validation=disabled
+    asserts:
+      - equal:
+          path: webhooks[0].objectSelector.matchExpressions[0].key
+          value: trainer.kubeflow.org/webhook-validation
+      - equal:
+          path: webhooks[0].objectSelector.matchExpressions[0].operator
+          value: NotIn
+      - contains:
+          path: webhooks[0].objectSelector.matchExpressions[0].values
+          content: disabled
+      - equal:
+          path: webhooks[1].objectSelector.matchExpressions[0].key
+          value: trainer.kubeflow.org/webhook-validation
+      - equal:
+          path: webhooks[1].objectSelector.matchExpressions[0].operator
+          value: NotIn
+      - contains:
+          path: webhooks[1].objectSelector.matchExpressions[0].values
+          content: disabled
+      - notExists:
+          path: webhooks[2].objectSelector
@@ -104,7 +104,7 @@ manager:
    runAsNonRoot: true
    capabilities:
      drop:
-      - ALL
+        - ALL
    seccompProfile:
      type: RuntimeDefault

@@ -188,6 +188,11 @@ runtimes:
  # Individual runtime settings will be ignored if this is enabled.
  defaultEnabled: false

+  # -- Common labels applied to every built-in runtime.
+  # The built-in ClusterTrainingRuntime are validated ahead of time.
+  commonLabels:
+    trainer.kubeflow.org/webhook-validation: disabled
+
  # -- PyTorch distributed training runtime (no custom images required)
  torchDistributed:
    # -- Enable deployment of torch-distributed runtime
@@ -128,6 +128,27 @@ BuiltinTrainers.

 Check [this guide](../user-guides/builtin-trainer/overview.md) to understand what is CustomTrainer and BuiltinTrainer.

+## Skipping Webhook Validation
+
+Kubeflow Trainer runs a validation webhook that verifies every `ClusterTrainingRuntime` and
+`TrainingRuntime` when it is created or updated. The built-in runtimes maintained by the Kubeflow
+community are validated ahead of time, so they carry the following label to skip this webhook:
+
+```yaml
+trainer.kubeflow.org/webhook-validation: disabled
+```
+
+This allows the runtimes to be deployed before the webhook server is ready, for example
+during a single-step `helm install` where the CRDs, control plane, and runtimes are applied
+together.
+
+:::{warning}
+This label opts the runtime out of webhook validation entirely, even after the webhook server is
+ready. It is intended only for trusted runtimes that are shipped and validated by the Kubeflow
+community. Runtimes that you create or modify should omit this label so that they are always
+validated.
+:::
+
 ## Supported Runtimes

 Kubeflow Trainer community maintains
@@ -137,20 +158,20 @@ platform administrators to extend these runtimes to fit their specific requireme

 The following runtimes are supported for CustomTrainer:

-| Runtime Name | ML Framework |
+| Runtime Name          | ML Framework                                                  |
 | --------------------- | ------------------------------------------------------------- |
-| torch-distributed | [PyTorch](https://docs.pytorch.org/docs/stable/index.html) |
-| deepspeed-distributed | [DeepSpeed](https://www.deepspeed.ai/) |
-| jax-distributed | [JAX](https://jax.readthedocs.io/) |
-| mlx-distributed | [MLX](https://ml-explore.github.io/mlx/build/html/index.html) |
-| xgboost-distributed | [XGBoost](https://xgboost.readthedocs.io/) |
+| torch-distributed     | [PyTorch](https://docs.pytorch.org/docs/stable/index.html)    |
+| deepspeed-distributed | [DeepSpeed](https://www.deepspeed.ai/)                        |
+| jax-distributed       | [JAX](https://jax.readthedocs.io/)                            |
+| mlx-distributed       | [MLX](https://ml-explore.github.io/mlx/build/html/index.html) |
+| xgboost-distributed   | [XGBoost](https://xgboost.readthedocs.io/)                    |

 The following runtimes are supported for TorchTune BuiltinTrainer:

-| Runtime Name | Pre-trained LLM |
-| --------------------- | --------------- |
-| torchtune-llama3.2-1b | Llama 3.2 (1B) |
-| torchtune-llama3.2-3b | Llama 3.2 (3B) |
+| Runtime Name           | Pre-trained LLM |
+| ---------------------- | --------------- |
+| torchtune-llama3.2-1b  | Llama 3.2 (1B)  |
+| torchtune-llama3.2-3b  | Llama 3.2 (3B)  |
 | torchtune-qwen2.5-1.5b | Qwen 2.5 (1.5B) |

 ### Runtime Deprecation Policy
@@ -4,6 +4,7 @@ metadata:
  name: flux-distributed
  labels:
    trainer.kubeflow.org/framework: flux
+    trainer.kubeflow.org/webhook-validation: disabled
 spec:
  mlPolicy:
    numNodes: 1
@@ -135,6 +135,16 @@ TRAINER_CI_IMAGE_NAME="ghcr.io/kubeflow/trainer/torchtune-trainer"
 TRAINER_CI_IMAGE="${TRAINER_CI_IMAGE_NAME}:${CI_IMAGE_TAG}"
 ${CONTAINER_RUNTIME} build . -f cmd/trainers/torchtune/Dockerfile -t ${TRAINER_CI_IMAGE}

+MLX_RUNTIME_CI_IMAGE_NAME="ghcr.io/kubeflow/trainer/mlx-runtime"
+MLX_RUNTIME_CI_IMAGE="${MLX_RUNTIME_CI_IMAGE_NAME}:${CI_IMAGE_TAG}"
+echo "Build MLX runtime image"
+${CONTAINER_RUNTIME} build . -f cmd/runtimes/mlx/Dockerfile -t ${MLX_RUNTIME_CI_IMAGE}
+
+DEEPSPEED_RUNTIME_CI_IMAGE_NAME="ghcr.io/kubeflow/trainer/deepspeed-runtime"
+DEEPSPEED_RUNTIME_CI_IMAGE="${DEEPSPEED_RUNTIME_CI_IMAGE_NAME}:${CI_IMAGE_TAG}"
+echo "Build DeepSpeed runtime image"
+${CONTAINER_RUNTIME} build . -f cmd/runtimes/deepspeed/Dockerfile -t ${DEEPSPEED_RUNTIME_CI_IMAGE}
+
 XGBOOST_RUNTIME_CI_IMAGE_NAME="ghcr.io/kubeflow/trainer/xgboost-runtime"
 XGBOOST_RUNTIME_CI_IMAGE="${XGBOOST_RUNTIME_CI_IMAGE_NAME}:${CI_IMAGE_TAG}"
 echo "Build XGBoost runtime image"
@@ -149,6 +159,8 @@ load_image_to_kind "${CONTROLLER_MANAGER_CI_IMAGE}" "${CLUSTER_NAME}"
 load_image_to_kind "${DATASET_INITIALIZER_CI_IMAGE}" "${CLUSTER_NAME}"
 load_image_to_kind "${MODEL_INITIALIZER_CI_IMAGE}" "${CLUSTER_NAME}"
 load_image_to_kind "${TRAINER_CI_IMAGE}" "${CLUSTER_NAME}"
+load_image_to_kind "${MLX_RUNTIME_CI_IMAGE}" "${CLUSTER_NAME}"
+load_image_to_kind "${DEEPSPEED_RUNTIME_CI_IMAGE}" "${CLUSTER_NAME}"
 load_image_to_kind "${XGBOOST_RUNTIME_CI_IMAGE}" "${CLUSTER_NAME}"
 load_image_to_kind "${JAX_RUNTIME_IMAGE}" "${CLUSTER_NAME}"

@@ -198,16 +210,20 @@ EOF
  apiVersion: kustomize.config.k8s.io/v1beta1
  kind: Kustomization
  resources:
-  - ../../../manifests/overlays/runtimes
+  - ../../../manifests/base/runtimes
  images:
-  - name: "${XGBOOST_RUNTIME_CI_IMAGE_NAME}"
-    newTag: "${CI_IMAGE_TAG}"
  - name: "${DATASET_INITIALIZER_CI_IMAGE_NAME}"
    newTag: "${CI_IMAGE_TAG}"
  - name: "${MODEL_INITIALIZER_CI_IMAGE_NAME}"
    newTag: "${CI_IMAGE_TAG}"
  - name: "${TRAINER_CI_IMAGE_NAME}"
    newTag: "${CI_IMAGE_TAG}"
+  - name: "${MLX_RUNTIME_CI_IMAGE_NAME}"
+    newTag: "${CI_IMAGE_TAG}"
+  - name: "${DEEPSPEED_RUNTIME_CI_IMAGE_NAME}"
+    newTag: "${CI_IMAGE_TAG}"
+  - name: "${XGBOOST_RUNTIME_CI_IMAGE_NAME}"
+    newTag: "${CI_IMAGE_TAG}"
 EOF

  kubectl apply --server-side -k "${E2E_RUNTIMES_DIR}" || (
@@ -228,8 +244,9 @@ elif [ "${INSTALL_METHOD}" = "helm" ]; then
    --namespace ${NAMESPACE} \
    --create-namespace \
    --set runtimes.defaultEnabled=true \
-    --set runtimes.xgboost.image.repository=${XGBOOST_RUNTIME_CI_IMAGE_NAME} \
-    --set runtimes.xgboost.image.tag=${CI_IMAGE_TAG} \
+    --set runtimes.xgboostDistributed.image.tag=${CI_IMAGE_TAG} \
+    --set runtimes.mlxDistributed.image.tag=${CI_IMAGE_TAG} \
+    --set runtimes.deepspeedDistributed.image.tag=${CI_IMAGE_TAG} \
    --set image.tag=${CI_IMAGE_TAG} \
    --set manager.config.featureGates.TrainJobStatus=true \
    --wait
@@ -4,6 +4,7 @@ metadata:
  name: torch-distributed-with-cache
  labels:
    trainer.kubeflow.org/framework: torch
+    trainer.kubeflow.org/webhook-validation: disabled
 spec:
  mlPolicy:
    numNodes: 1
@@ -10,3 +10,19 @@
 - op: replace
  path: /metadata/name
  value: validator.trainer.kubeflow.org
+- op: add
+  path: /webhooks/0/objectSelector
+  value:
+    matchExpressions:
+      - key: trainer.kubeflow.org/webhook-validation
+        operator: NotIn
+        values:
+          - disabled
+- op: add
+  path: /webhooks/1/objectSelector
+  value:
+    matchExpressions:
+      - key: trainer.kubeflow.org/webhook-validation
+        operator: NotIn
+        values:
+          - disabled
@@ -3,6 +3,12 @@ kind: Kustomization
 resources:
  - ../../base/runtimes

+# The built-in ClusterTrainingRuntime are validated ahead of time.
+labels:
+  - pairs:
+      trainer.kubeflow.org/webhook-validation: disabled
+    includeSelectors: false
+
 # Update the Kubeflow LLM Trainer image tag.
 images:
  - name: ghcr.io/kubeflow/trainer/torchtune-trainer