spark-operator/charts/spark-operator-chart/values.yaml

#
# Copyright 2024 The Kubeflow authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# Default values for spark-operator.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.

# -- String to partially override release name.
nameOverride: ""

# -- String to fully override release name.
fullnameOverride: ""

# -- Common labels to add to the resources.
commonLabels: {}

# Image used by the Spark operator.
image:
  # -- Image registry.
  registry: ghcr.io
  # -- Image repository.
  repository: kubeflow/spark-operator/controller
  # -- Image tag.
  # @default -- If not set, the chart appVersion will be used.
  tag: ""
  # -- Image pull policy.
  pullPolicy: IfNotPresent
  # -- Image pull secrets for private image registry.
  pullSecrets: []
  # - name: <secret-name>

# Helm hook configuration.
hook:
  # -- Whether to create a Helm pre-install/pre-upgrade hook Job to update CRDs.
  upgradeCrd: false
  # Image used by the Helm hook Job.
  image:
    # -- Image registry.
    registry: ghcr.io
    # -- Image repository.
    repository: kubeflow/spark-operator/kubectl
    # -- Image tag.
    # @default -- If not set, the chart appVersion will be used.
    tag: ""
  # -- Pod resource requests and limits for the Helm hook Job container.
  resources:
    requests:
      cpu: 100m
      memory: 64Mi
    limits:
      cpu: 100m
      memory: 64Mi

  # -- Node selector for the Helm hook Job.
  nodeSelector: {}

  # -- Affinity for the Helm hook Job.
  affinity: {}

  # -- List of node taints to tolerate for the Helm hook Job.
  tolerations: []

  # -- Extra labels for the Helm hook Job pod.
  labels: {}
    # key1: value1
    # key2: value2

  # -- Extra annotations for the Helm hook Job pod.
  annotations: {}
    # key1: value1
    # key2: value2

controller:
  # -- Number of replicas of controller.
  replicas: 1

  # -- Feature gates to enable or disable specific features.
  featureGates:
  - name: PartialRestart
    enabled: false
  - name: LoadSparkDefaults
    enabled: false
  - name: RestSubmitter
    enabled: false

  # -- The number of old history to retain to allow rollback.
  revisionHistoryLimit: 10

  leaderElection:
    # -- Specifies whether to enable leader election for controller.
    enable: true
    # -- Leader election lease duration.
    leaseDuration: 15s
    # -- Leader election renew deadline.
    renewDeadline: 10s
    # -- Leader election retry period.
    retryPeriod: 2s

  # -- Reconcile concurrency, higher values might increase memory usage.
  workers: 10

  # -- Configure the verbosity of logging, can be one of `debug`, `info`, `error`.
  logLevel: info

  # -- Configure the encoder of logging, can be one of `console` or `json`.
  logEncoder: console

  # -- Grace period after a successful spark-submit when driver pod not found errors will be retried. Useful if the driver pod can take some time to be created.
  driverPodCreationGracePeriod: 10s

  # -- Specifies the maximum number of Executor pods that can be tracked by the controller per SparkApplication.
  maxTrackedExecutorPerApp: 1000

  # -- Driver PDB feature gate. When true, the controller creates a PodDisruptionBudget for each SparkApplication that sets spec.driverPodDisruptionBudget=true. Default false.
  driverPodDisruptionBudget:
    enable: false

  # -- Maximum QPS to the API server from the controller client.
  kubeAPIQPS: 20
  # -- Maximum burst for throttle from the controller client.
  kubeAPIBurst: 30

  # -- Timestamp precision for ScheduledSparkApplication run names.
  # Valid values: nanos (default), micros, millis, seconds, minutes.
  # Shorter precisions produce shorter names which helps with Kubernetes name length limits.
  # NOTE: Using lower precisions such as "seconds" or "minutes" increases the risk of name
  # collisions if multiple runs are created within the same time unit (for example during
  # reconciliation loops or manual re-triggers). A collision will cause run creation to fail.
  # Choose a precision compatible with your scheduling frequency: "minutes" is only suitable
  # for jobs scheduled at most once per minute, "seconds" for jobs scheduled at most once per second.
  scheduledSparkApplicationTimestampPrecision: nanos

  uiService:
    # -- Specifies whether to create service for Spark web UI.
    enable: true

  uiIngress:
    # -- Specifies whether to create ingress for Spark web UI.
    # `controller.uiService.enable` must be `true` to enable ingress.
    enable: false
    # -- Ingress URL format.
    # Required if `controller.uiIngress.enable` is true.
    urlFormat: ""
    # -- Optionally set the ingressClassName.
    ingressClassName: ""
    # -- Optionally set default TLS configuration for the Spark UI's ingress. `ingressTLS` in the SparkApplication spec overrides this.
    tls: []
    # - hosts:
    #   - "*.example.com"
    #   secretName: "example-secret"
    # -- Optionally set default ingress annotations for the Spark UI's ingress. `ingressAnnotations` in the SparkApplication spec overrides this.
    annotations: {}
      # key1: value1
      # key2: value2

  batchScheduler:
    # -- Specifies whether to enable batch scheduler for spark jobs scheduling.
    # If enabled, users can specify batch scheduler name in spark application.
    enable: false
    # -- Specifies a list of kube-scheduler names for scheduling Spark pods.
    kubeSchedulerNames: []
    # - default-scheduler
    # -- Default batch scheduler to be used if not specified by the user.
    # If specified, this value must be either "volcano" or "yunikorn". Specifying any other
    # value will cause the controller to error on startup.
    default: ""

  serviceAccount:
    # -- Specifies whether to create a service account for the controller.
    create: true
    # -- Optional name for the controller service account.
    name: ""
    # -- Extra annotations for the controller service account.
    annotations: {}
    # -- Auto-mount service account token to the controller pods.
    automountServiceAccountToken: true

  rbac:
    # -- Specifies whether to create RBAC resources for the controller.
    create: true
    # -- Extra annotations for the controller RBAC resources.
    annotations: {}

  # -- Extra labels for controller pods.
  labels: {}
    # key1: value1
    # key2: value2

  # -- Extra annotations for controller pods.
  annotations: {}
    # key1: value1
    # key2: value2

  # -- Annotations for the controller Deployment resource (metadata.annotations).
  deploymentAnnotations: {}
    # key1: value1
    # key2: value2

  # -- Volumes for controller pods.
  volumes:
  # Create a tmp directory to write Spark artifacts to for deployed Spark apps.
  - name: tmp
    emptyDir:
      sizeLimit: 1Gi

  # -- Node selector for controller pods.
  nodeSelector: {}

  # -- Affinity for controller pods.
  affinity: {}

  # -- List of node taints to tolerate for controller pods.
  tolerations: []

  # -- Priority class for controller pods.
  priorityClassName: ""

  # -- Security context for controller pods.
  podSecurityContext:
    fsGroup: 185

  # -- Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in.
  # Ref: [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/).
  # The labelSelector field in topology spread constraint will be set to the selector labels for controller pods if not specified.
  topologySpreadConstraints: []
  # - maxSkew: 1
  #   topologyKey: topology.kubernetes.io/zone
  #   whenUnsatisfiable: ScheduleAnyway
  # - maxSkew: 1
  #   topologyKey: kubernetes.io/hostname
  #   whenUnsatisfiable: DoNotSchedule

  # -- Whether to use user namespace or not
  # Kubernetes version 1.30 for feature beta (1.33 for GA) or higher is required with support from OS and OCI runtime
  # ref: https://kubernetes.io/docs/concepts/workloads/pods/user-namespaces/
  hostUsers: null

  # -- Environment variables for controller containers.
  env: []

  # -- Environment variable sources for controller containers.
  envFrom: []

  # -- Volume mounts for controller containers.
  volumeMounts:
  # Mount a tmp directory to write Spark artifacts to for deployed Spark apps.
  - name: tmp
    mountPath: "/tmp"
    readOnly: false

  # -- Pod resource requests and limits for controller containers.
  # Note, that each job submission will spawn a JVM within the controller pods using "/usr/local/openjdk-11/bin/java -Xmx128m".
  # Kubernetes may kill these Java processes at will to enforce resource limits. When that happens, you will see the following error:
  # 'failed to run spark-submit for SparkApplication [...]: signal: killed' - when this happens, you may want to increase memory limits.
  resources: {}
    # limits:
    #   cpu: 100m
    #   memory: 300Mi
    # requests:
    #   cpu: 100m
    #   memory: 300Mi

  # -- Security context for controller containers.
  securityContext:
    readOnlyRootFilesystem: true
    privileged: false
    allowPrivilegeEscalation: false
    runAsNonRoot: true
    capabilities:
      drop:
      - ALL
    seccompProfile:
      type: RuntimeDefault

  # -- Sidecar containers for controller pods.
  sidecars: []

  # Pod disruption budget for controller to avoid service degradation.
  podDisruptionBudget:
    # -- Specifies whether to create pod disruption budget for controller.
    # Ref: [Specifying a Disruption Budget for your Application](https://kubernetes.io/docs/tasks/run-application/configure-pdb/)
    enable: false
    # -- The number of pods that must be available.
    # Require `controller.replicas` to be greater than 1
    minAvailable: 1

  pprof:
    # -- Specifies whether to enable pprof.
    enable: false
    # -- Specifies pprof port.
    port: 6060
    # -- Specifies pprof service port name.
    portName: pprof

  # Workqueue rate limiter configuration forwarded to the controller-runtime Reconciler.
  workqueueRateLimiter:
    # -- Specifies the average rate of items process by the workqueue rate limiter.
    bucketQPS: 50
    # -- Specifies the maximum number of items that can be in the workqueue at any given time.
    bucketSize: 500
    maxDelay:
      # -- Specifies whether to enable max delay for the workqueue rate limiter.
      # This is useful to avoid losing events when the workqueue is full.
      enable: true
      # -- Specifies the maximum delay duration for the workqueue rate limiter.
      duration: 6h

# REST Submitter Service configuration.
# The submitter service itself is deployed separately (see submitter service repository).
# These values configure the controller's connection to the submitter.
# Enable via controller.featureGates RestSubmitter=true.
submitter:
  # -- URL of the REST submitter service endpoint.
  serviceUrl: http://spark-operator-submitter-svc:8080/api/v1/spark-submit

  # -- HTTP request timeout per spark submission attempt.
  requestTimeout: 2m

  # -- How long the controller waits for the submitter service to become reachable at startup.
  startupTimeout: 5m

  # -- Max retry attempts for transient submission failures.
  maxRetries: 3

  # -- Initial backoff duration before the first retry. Doubles on each subsequent attempt.
  initialBackoff: 2s

  # Controller startup probe settings. Allows the controller to block during startup
  # without being killed by liveness probe. Should cover startupTimeout.
  startupProbe:
    # -- Seconds between probe attempts.
    periodSeconds: 5
    # -- Number of failures before restart. (failureThreshold × periodSeconds) should >= startupTimeout.
    failureThreshold: 60

webhook:
  # -- Specifies whether to enable webhook.
  enable: true

  # -- Number of replicas of webhook server.
  replicas: 1

  # -- The number of old history to retain to allow rollback.
  revisionHistoryLimit: 10

  leaderElection:
    # -- Specifies whether to enable leader election for webhook.
    enable: true

  # -- Configure the verbosity of logging, can be one of `debug`, `info`, `error`.
  logLevel: info

  # -- Configure the encoder of logging, can be one of `console` or `json`.
  logEncoder: console

  # -- Specifies webhook port.
  port: 9443

  # -- Specifies webhook service port name.
  portName: webhook

  # -- Specifies how unrecognized errors are handled.
  # Available options are `Ignore` or `Fail`.
  failurePolicy: Fail

  # -- Specifies the timeout seconds of the webhook, the value must be between 1 and 30.
  timeoutSeconds: 10

  # -- Maximum QPS to the API server from the controller client.
  kubeAPIQPS: 20
  # -- Maximum burst for throttle from the controller client.
  kubeAPIBurst: 30

  resourceQuotaEnforcement:
    # -- Specifies whether to enable the ResourceQuota enforcement for SparkApplication resources.
    enable: false

  serviceAccount:
    # -- Specifies whether to create a service account for the webhook.
    create: true
    # -- Optional name for the webhook service account.
    name: ""
    # -- Extra annotations for the webhook service account.
    annotations: {}
    # -- Auto-mount service account token to the webhook pods.
    automountServiceAccountToken: true

  rbac:
    # -- Specifies whether to create RBAC resources for the webhook.
    create: true
    # -- Extra annotations for the webhook RBAC resources.
    annotations: {}

  # -- Extra labels for webhook pods.
  labels: {}
    # key1: value1
    # key2: value2

  # -- Extra annotations for webhook pods.
  annotations: {}
    # key1: value1
    # key2: value2

  # -- Annotations for the webhook Deployment resource (metadata.annotations).
  deploymentAnnotations: {}
    # key1: value1
    # key2: value2

  # -- Sidecar containers for webhook pods.
  sidecars: []

  # -- Volumes for webhook pods.
  volumes:
  # Create a dir for the webhook to generate its certificates in.
  - name: serving-certs
    emptyDir:
      sizeLimit: 500Mi

  # -- Node selector for webhook pods.
  nodeSelector: {}

  # -- Affinity for webhook pods.
  affinity: {}

  # -- List of node taints to tolerate for webhook pods.
  tolerations: []

  # -- Priority class for webhook pods.
  priorityClassName: ""

  # -- Security context for webhook pods.
  podSecurityContext:
    fsGroup: 185

  # -- Topology spread constraints rely on node labels to identify the topology domain(s) that each Node is in.
  # Ref: [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/).
  # The labelSelector field in topology spread constraint will be set to the selector labels for webhook pods if not specified.
  topologySpreadConstraints: []
  # - maxSkew: 1
  #   topologyKey: topology.kubernetes.io/zone
  #   whenUnsatisfiable: ScheduleAnyway
  # - maxSkew: 1
  #   topologyKey: kubernetes.io/hostname
  #   whenUnsatisfiable: DoNotSchedule

  # -- Whether to use user namespace or not
  # Kubernetes version 1.30 for feature beta (1.33 for GA) or higher is required with support from OS and OCI runtime
  # ref: https://kubernetes.io/docs/concepts/workloads/pods/user-namespaces/
  hostUsers: null

  # -- Environment variables for webhook containers.
  env: []

  # -- Environment variable sources for webhook containers.
  envFrom: []

  # -- Volume mounts for webhook containers.
  volumeMounts:
  # Mount a dir for the webhook to generate its certificates in.
  - name: serving-certs
    mountPath: /etc/k8s-webhook-server/serving-certs
    subPath: serving-certs
    readOnly: false


  # -- Pod resource requests and limits for webhook pods.
  resources: {}
    # limits:
    #   cpu: 100m
    #   memory: 300Mi
    # requests:
    #   cpu: 100m
    #   memory: 300Mi

  # -- Security context for webhook containers.
  securityContext:
    readOnlyRootFilesystem: true
    privileged: false
    allowPrivilegeEscalation: false
    runAsNonRoot: true
    capabilities:
      drop:
      - ALL
    seccompProfile:
      type: RuntimeDefault

  # Pod disruption budget for webhook to avoid service degradation.
  podDisruptionBudget:
    # -- Specifies whether to create pod disruption budget for webhook.
    # Ref: [Specifying a Disruption Budget for your Application](https://kubernetes.io/docs/tasks/run-application/configure-pdb/)
    enable: false
    # -- The number of pods that must be available.
    # Require `webhook.replicas` to be greater than 1
    minAvailable: 1

spark:
  # -- List of namespaces where to run spark jobs.
  # If empty string is included, all namespaces will be allowed.
  # Namespaces specified here will be watched in addition to those matching jobNamespaceSelector.
  # Make sure the namespaces have already existed.
  jobNamespaces:
  - default

  # -- Label selector to filter namespaces to watch.
  # Supports standard Kubernetes label selector syntax (e.g., 'spark-operator=enabled,env in (prod,staging)').
  # Namespaces matching this selector will be watched in addition to those in jobNamespaces.
  # When specified, requires ClusterRole permission to list and watch namespaces.
  # Leave empty to disable namespace selector functionality.
  jobNamespaceSelector: ""

  serviceAccount:
    # -- Specifies whether to create a service account for spark applications.
    create: true
    # -- Optional name for the spark service account.
    name: ""
    # -- Optional annotations for the spark service account.
    annotations: {}
    # -- Auto-mount service account token to the spark applications pods.
    automountServiceAccountToken: true

  rbac:
    # -- Specifies whether to create RBAC resources for spark applications.
    create: true
    # -- Optional annotations for the spark application RBAC resources.
    annotations: {}

prometheus:
  metrics:
    # -- Specifies whether to enable prometheus metrics scraping.
    enable: true
    # -- Metrics port.
    port: 8080
    # -- Metrics port name.
    portName: metrics
    # -- Metrics serving endpoint.
    endpoint: /metrics
    # -- Metrics prefix, will be added to all exported metrics.
    prefix: ""
    # -- Job Submit Latency histogram buckets. Specified in seconds.
    jobSubmitLatencyBuckets: "0.5,1,2,4,8,16,32,64,128,256"
    # -- Job Start Latency histogram buckets. Specified in seconds.
    jobStartLatencyBuckets: "30,60,90,120,150,180,210,240,270,300"
    # -- Labels to be added to the Spark Operator standard metrics, e.g., "label1Key,label2Key".
    # Defaults to 'app_type' if not set.
    labels: ""

  # Prometheus pod monitor for controller pods
  podMonitor:
    # -- Specifies whether to create pod monitor.
    # Note that prometheus metrics should be enabled as well.
    create: false
    # -- Pod monitor labels
    labels: {}
    # -- The label to use to retrieve the job name from
    jobLabel: spark-operator-podmonitor
    # -- Prometheus metrics endpoint properties. `metrics.portName` will be used as a port
    podMetricsEndpoint:
      scheme: http
      interval: 5s

certManager:
  # -- Specifies whether to use [cert-manager](https://cert-manager.io) to generate certificate for webhook.
  # `webhook.enable` must be set to `true` to enable cert-manager.
  enable: false
  # -- The reference to the issuer.
  # @default -- A self-signed issuer will be created and used if not specified.
  issuerRef: {}
    # group: cert-manager.io
    # kind: ClusterIssuer
    # name: selfsigned
  # -- The duration of the certificate validity (e.g. `2160h`).
  # See [cert-manager.io/v1.Certificate](https://cert-manager.io/docs/reference/api-docs/#cert-manager.io/v1.Certificate).
  # @default -- `2160h` (90 days) will be used if not specified.
  duration: ""
  # -- The duration before the certificate expiration to renew the certificate (e.g. `720h`).
  # See [cert-manager.io/v1.Certificate](https://cert-manager.io/docs/reference/api-docs/#cert-manager.io/v1.Certificate).
  # @default -- 1/3 of issued certificate’s lifetime.
  renewBefore: ""