mirror of
https://github.com/hanzo-ml/spark-operator.git
synced 2026-10-30 20:39:07 +00:00
642f7381b1
* feat: optional PodDisruptionBudget for Spark drivers (#2949) Refs: https://github.com/kubeflow/spark-operator/issues/2949 Add an opt-in PodDisruptionBudget for the driver pod so a voluntary eviction (e.g. `kubectl drain`) cannot kill the driver mid-job. The apiserver's eviction admission handler consults a matching PDB; without one the eviction is admitted and the operator only learns of it after the fact via its watch. A PDB is the only lever the operator has here that does not put it on the eviction hot path. Gated twice, off by default so upgrades are no-op: --enable-driver-pdb controller flag (default false) spec.driverPodDisruptionBudget per-app *bool (nil/false = no PDB) A bare *bool suffices: the driver is a singleton, so MinAvailable is fixed at 1 and a selector struct has nothing to grow into. Lifecycle (driver_pdb.go): - ensureDriverPDB converges PDB existence to spec on every Submitted/Running reconcile, so it recovers from operator restart with the gate flipped on, live spec edits, and transient API failures (retry.RetryOnConflict + controller-runtime backoff). - createDriverPDB/deleteDriverPDB read the cache first for idempotency; AlreadyExists/NotFound are treated as success. - The PDB carries a controller OwnerReference plus LabelLaunchedBySparkOperator + LabelSparkAppName, so kube GC reaps it on app deletion and the cache informer can filter to owned PDBs. - cleanUpOnTermination deletes the PDB on Completed/Failed; the terminal-app reconcile re-runs it to reap PDBs orphaned when the controller itself was drained mid-transition. - deleteSparkResources drops the PDB before the driver pod so an in-flight external drain does not block on a stale PDB. RBAC: the PDB entry in cache.Options.ByObject starts an informer, which needs list+watch even though the code only Get/Create/Deletes. Granted get;list;watch;create;delete on poddisruptionbudgets via the kubebuilder marker (regenerated config/rbac/role.yaml), the Helm chart helper, and the chart rbac unit test. Missing verbs would loop the informer on Forbidden and saturate the shared client rate limiter. Helm: controller.driverPodDisruptionBudget.enable (default false) emits --enable-driver-pdb=true; ci-values.yaml turns it on for e2e. Tests: envtest specs cover the gate matrix and the convergence cases ensureDriverPDB owns (restart-with-gate-on, spec false->true and true->false on a running app, transient Create failure retried via an interceptor.Funcs client). E2e submits real apps with the field set and omitted and asserts PDB creation, GC on deletion, and absence when not opted in. Makefile: kind-load-spark-image preloads the Spark runtime image and e2e-test pins IMAGE_TAG=local with the load targets as prerequisites, so `make deploy` and `make e2e-test` are self-contained on a local Kind cluster instead of hitting ImagePullBackOff. Signed-off-by: Vincent Janelle <[email protected]> * test: use events.FakeRecorder in driver PDB test NewReconciler takes an events.EventRecorder, not the legacy record.EventRecorder. Align driver_pdb_test.go with controller_test.go. Signed-off-by: Vincent Janelle <[email protected]> * allow for kustomize to patch pdb value; default to true in ci Signed-off-by: Vincent Janelle <[email protected]> --------- Signed-off-by: Vincent Janelle <[email protected]> Co-authored-by: Hanzo Dev <[email protected]>
95 lines
2.8 KiB
Go
95 lines
2.8 KiB
Go
/*
|
|
Copyright 2024 The Kubeflow authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
https://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package sparkapplication_test
|
|
|
|
import (
|
|
"fmt"
|
|
"path/filepath"
|
|
"runtime"
|
|
"testing"
|
|
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
|
|
policyv1 "k8s.io/api/policy/v1"
|
|
"k8s.io/client-go/kubernetes/scheme"
|
|
"k8s.io/client-go/rest"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/envtest"
|
|
"sigs.k8s.io/controller-runtime/pkg/log"
|
|
"sigs.k8s.io/controller-runtime/pkg/log/zap"
|
|
|
|
"github.com/kubeflow/spark-operator/v2/api/v1beta2"
|
|
// +kubebuilder:scaffold:imports
|
|
)
|
|
|
|
// These tests use Ginkgo (BDD-style Go testing framework). Refer to
|
|
// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
|
|
|
|
var cfg *rest.Config
|
|
var k8sClient client.Client
|
|
var testEnv *envtest.Environment
|
|
|
|
func TestControllers(t *testing.T) {
|
|
RegisterFailHandler(Fail)
|
|
|
|
RunSpecs(t, "Controller Suite")
|
|
}
|
|
|
|
var _ = BeforeSuite(func() {
|
|
log.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true)))
|
|
|
|
By("bootstrapping test environment")
|
|
testEnv = &envtest.Environment{
|
|
CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")},
|
|
ErrorIfCRDPathMissing: true,
|
|
|
|
// The BinaryAssetsDirectory is only required if you want to run the tests directly
|
|
// without call the makefile target test. If not informed it will look for the
|
|
// default path defined in controller-runtime which is /usr/local/kubebuilder/.
|
|
// Note that you must have the required binaries setup under the bin directory to perform
|
|
// the tests directly. When we run make test it will be setup and used automatically.
|
|
BinaryAssetsDirectory: filepath.Join("..", "..", "..", "bin", "k8s",
|
|
fmt.Sprintf("1.35.0-%s-%s", runtime.GOOS, runtime.GOARCH)),
|
|
}
|
|
|
|
var err error
|
|
// cfg is defined in this file globally.
|
|
cfg, err = testEnv.Start()
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(cfg).NotTo(BeNil())
|
|
|
|
err = v1beta2.AddToScheme(scheme.Scheme)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
err = policyv1.AddToScheme(scheme.Scheme)
|
|
Expect(err).NotTo(HaveOccurred())
|
|
|
|
// +kubebuilder:scaffold:scheme
|
|
|
|
k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
|
|
Expect(err).NotTo(HaveOccurred())
|
|
Expect(k8sClient).NotTo(BeNil())
|
|
|
|
})
|
|
|
|
var _ = AfterSuite(func() {
|
|
By("tearing down the test environment")
|
|
err := testEnv.Stop()
|
|
Expect(err).NotTo(HaveOccurred())
|
|
})
|