hanzo-ml/kserve
2
0
Fork 0
mirror of https://github.com/hanzo-ml/kserve.git synced 2026-10-30 20:53:49 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
master
kserve/pkg/controller/v1alpha2/llmisvc/config_merge_test.go
T
Andres LlausasGitHubClaude Sonnet 4.6
 15a92ece5c feat(llmisvc): add kvCacheOffloading spec for CPU KV cache tiering (#5599) 
Signed-off-by: Andres Llausas <[email protected]>
Co-authored-by: Claude Sonnet 4.6 <[email protected]>
2026-06-24 09:21:34 -07:00

2888 lines
81 KiB
Go
Raw Permalink Blame History

/*
Copyright 2025 The KServe Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package llmisvc_test
import (
"strings"
"testing"
. "github.com/onsi/gomega"
"sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/yaml"
ktesting "github.com/kserve/kserve/pkg/testing"
"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc"
"github.com/kserve/kserve/pkg/credentials"
"github.com/google/go-cmp/cmp"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"
"knative.dev/pkg/apis"
duckv1 "knative.dev/pkg/apis/duck/v1"
igwapi "sigs.k8s.io/gateway-api-inference-extension/api/v1"
gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
"github.com/kserve/kserve/pkg/apis/serving/v1alpha2"
pkgtest "github.com/kserve/kserve/pkg/testing"
)
func TestMergeSpecs(t *testing.T) {
tests := []struct {
name string
cfgs []v1alpha2.LLMInferenceServiceSpec
want v1alpha2.LLMInferenceServiceSpec
wantErr bool
}{
{
name: "no configs",
cfgs: []v1alpha2.LLMInferenceServiceSpec{},
want: v1alpha2.LLMInferenceServiceSpec{},
wantErr: false,
},
{
name: "single config",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{Model: v1alpha2.LLMModelSpec{URI: apis.URL{Path: "model-a"}}},
},
want: v1alpha2.LLMInferenceServiceSpec{Model: v1alpha2.LLMModelSpec{URI: apis.URL{Path: "model-a"}}},
wantErr: false,
},
{
name: "two configs simple merge",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{Model: v1alpha2.LLMModelSpec{URI: apis.URL{Path: "model-a"}}},
},
want: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{URI: apis.URL{Path: "model-a"}},
},
wantErr: false,
},
{
name: "two configs simple merge with sub-field override",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{
Model: v1alpha2.LLMModelSpec{URI: apis.URL{Path: "model-a"}},
Router: &v1alpha2.RouterSpec{
Route: &v1alpha2.GatewayRoutesSpec{},
Gateway: &v1alpha2.GatewaySpec{},
Scheduler: &v1alpha2.SchedulerSpec{},
},
},
{
Model: v1alpha2.LLMModelSpec{URI: apis.URL{Path: "model-a"}},
Router: &v1alpha2.RouterSpec{
Scheduler: &v1alpha2.SchedulerSpec{
Pool: &v1alpha2.InferencePoolSpec{
Spec: &igwapi.InferencePoolSpec{
TargetPorts: []igwapi.Port{{Number: 9999}},
},
},
},
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{URI: apis.URL{Path: "model-a"}},
Router: &v1alpha2.RouterSpec{
Route: &v1alpha2.GatewayRoutesSpec{},
Gateway: &v1alpha2.GatewaySpec{},
Scheduler: &v1alpha2.SchedulerSpec{
Pool: &v1alpha2.InferencePoolSpec{
Spec: &igwapi.InferencePoolSpec{
TargetPorts: []igwapi.Port{{Number: 9999}},
},
},
},
},
},
wantErr: false,
},
{
name: "two configs with override",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{
Model: v1alpha2.LLMModelSpec{URI: apis.URL{Path: "model-a"}},
WorkloadSpec: v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](1),
},
},
{
Model: v1alpha2.LLMModelSpec{URI: apis.URL{Path: "model-b"}},
WorkloadSpec: v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](2),
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{URI: apis.URL{Path: "model-b"}},
WorkloadSpec: v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](2),
},
},
wantErr: false,
},
{
name: "three configs chained merge",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{Model: v1alpha2.LLMModelSpec{URI: apis.URL{Path: "model-a"}}},
{
Model: v1alpha2.LLMModelSpec{URI: apis.URL{Path: "model-b"}},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{URI: apis.URL{Path: "model-b"}},
},
wantErr: false,
},
{
name: "deep merge with podspec template",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
// Base configuration
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](1),
Template: &corev1.PodSpec{
InitContainers: []corev1.Container{
{
Name: "storage-initializer",
Image: "kserve/storage-initializer:latest",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceStorage: resource.MustParse("1Mi"),
},
},
},
},
Containers: []corev1.Container{
{
Name: "kserve-container",
Image: "base:0.1",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("1"),
},
},
},
},
Tolerations: []corev1.Toleration{
{Key: "team", Operator: corev1.TolerationOpEqual, Value: "a"},
},
},
},
},
// Override configuration
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](2),
Template: &corev1.PodSpec{
InitContainers: []corev1.Container{
{
Name: "storage-initializer",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceStorage: resource.MustParse("1Gi"),
},
},
},
},
Containers: []corev1.Container{
// This container should replace the base one due to the same name
{
Name: "kserve-container",
Image: "override:1.0",
Env: []corev1.EnvVar{
{Name: "FOO", Value: "bar"},
},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("2"), // Override CPU
},
},
},
// This is a new container that should be added
{
Name: "transformer",
Image: "transformer:latest",
},
},
// Tolerations should be REPLACED, not merged, as there is no patchMergeKey
Tolerations: []corev1.Toleration{
{Key: "gpu", Operator: corev1.TolerationOpExists},
},
PriorityClassName: "high-priority", // Add a new field
},
},
},
},
// Expected result of the merge
want: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](2),
Template: &corev1.PodSpec{
InitContainers: []corev1.Container{
{
Name: "storage-initializer",
Image: "kserve/storage-initializer:latest", // Image is preserved from base
Resources: corev1.ResourceRequirements{ // Resources are updated from override
Requests: corev1.ResourceList{
corev1.ResourceStorage: resource.MustParse("1Gi"),
},
},
},
},
Containers: []corev1.Container{
{
Name: "kserve-container",
Image: "override:1.0",
Env: []corev1.EnvVar{
{Name: "FOO", Value: "bar"},
},
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("2"),
},
},
},
{
Name: "transformer",
Image: "transformer:latest",
},
},
// Tolerations slice is replaced by the override
Tolerations: []corev1.Toleration{
{Key: "gpu", Operator: corev1.TolerationOpExists},
},
PriorityClassName: "high-priority",
},
},
},
wantErr: false,
},
{
name: "merge with prefill spec",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
// Base has only a decode workload
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](1),
Template: &corev1.PodSpec{Containers: []corev1.Container{{Name: "kserve-container", Image: "decode:0.1"}}},
},
},
// Override adds a prefill workload
{
Prefill: &v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](4),
Template: &corev1.PodSpec{Containers: []corev1.Container{{Name: "kserve-container", Image: "prefill:0.1"}}},
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
// Base workload spec is preserved
WorkloadSpec: v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](1),
Template: &corev1.PodSpec{Containers: []corev1.Container{{Name: "kserve-container", Image: "decode:0.1"}}},
},
// Prefill spec is added
Prefill: &v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](4),
Template: &corev1.PodSpec{Containers: []corev1.Container{{Name: "kserve-container", Image: "prefill:0.1"}}},
},
},
},
{
name: "merge with worker spec",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
// Base has the main head/decode template
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{Containers: []corev1.Container{{Name: "kserve-container", Image: "head:0.1"}}},
},
},
// Override adds a worker spec
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Worker: &corev1.PodSpec{Containers: []corev1.Container{{Name: "kserve-container", Image: "worker:0.1"}}},
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
// Head template is preserved
Template: &corev1.PodSpec{Containers: []corev1.Container{{Name: "kserve-container", Image: "head:0.1"}}},
// Worker spec is added
Worker: &corev1.PodSpec{Containers: []corev1.Container{{Name: "kserve-container", Image: "worker:0.1"}}},
},
},
},
{
name: "merge with parallelism spec",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
// Base defines tensor parallelism
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Parallelism: &v1alpha2.ParallelismSpec{
Tensor: ptr.To[int32](2),
},
},
},
// Override defines pipeline parallelism
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Parallelism: &v1alpha2.ParallelismSpec{
Pipeline: ptr.To[int32](4),
},
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
// Both parallelism values should be present
Parallelism: &v1alpha2.ParallelismSpec{
Tensor: ptr.To[int32](2),
Pipeline: ptr.To[int32](4),
},
},
},
},
{
name: "deep merge of prefill spec",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
// Base defines a prefill workload with replicas and a container with a resource request
{
Prefill: &v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](2),
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "prefill-container",
Image: "prefill:0.1",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")},
},
},
},
},
},
},
// Override changes replica count and adds an environment variable to the container
{
Prefill: &v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](4),
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "prefill-container",
Env: []corev1.EnvVar{
{Name: "PREFILL_MODE", Value: "FAST"},
},
},
},
},
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
Prefill: &v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](4), // Replicas are overridden
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "prefill-container",
Image: "prefill:0.1", // Image is preserved from base
Env: []corev1.EnvVar{ // Env var is added from override
{Name: "PREFILL_MODE", Value: "FAST"},
},
Resources: corev1.ResourceRequirements{ // Resources are preserved from base
Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")},
},
},
},
},
},
},
},
{
name: "4 chained merge router, epp, multi node",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{
Router: &v1alpha2.RouterSpec{
Route: &v1alpha2.GatewayRoutesSpec{},
Gateway: &v1alpha2.GatewaySpec{},
},
},
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Parallelism: &v1alpha2.ParallelismSpec{
Tensor: ptr.To[int32](1),
Pipeline: ptr.To[int32](1),
},
Worker: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "main",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("1"),
"nvidia.com/gpu": resource.MustParse("1"),
},
},
},
},
},
},
},
{
Router: &v1alpha2.RouterSpec{
Scheduler: &v1alpha2.SchedulerSpec{
Pool: &v1alpha2.InferencePoolSpec{
Spec: &igwapi.InferencePoolSpec{
TargetPorts: nil,
EndpointPickerRef: igwapi.EndpointPickerRef{
FailureMode: igwapi.EndpointPickerFailClose,
},
},
},
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "main",
},
},
},
},
},
},
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Parallelism: &v1alpha2.ParallelismSpec{
Tensor: ptr.To[int32](4),
Pipeline: ptr.To[int32](2),
},
Worker: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "main",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("1"),
"nvidia.com/gpu": resource.MustParse("4"),
},
},
},
},
},
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
Router: &v1alpha2.RouterSpec{
Route: &v1alpha2.GatewayRoutesSpec{},
Gateway: &v1alpha2.GatewaySpec{},
Scheduler: &v1alpha2.SchedulerSpec{
Pool: &v1alpha2.InferencePoolSpec{
Spec: &igwapi.InferencePoolSpec{
TargetPorts: nil,
EndpointPickerRef: igwapi.EndpointPickerRef{
FailureMode: igwapi.EndpointPickerFailClose,
},
},
},
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "main",
},
},
},
},
},
WorkloadSpec: v1alpha2.WorkloadSpec{
Parallelism: &v1alpha2.ParallelismSpec{
Tensor: ptr.To[int32](4),
Pipeline: ptr.To[int32](2),
},
Worker: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "main",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("1"),
"nvidia.com/gpu": resource.MustParse("4"),
},
},
},
},
},
},
},
},
{
name: "4 chained merge router with scheduler, http route and gateway ref, multi node",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{
Router: &v1alpha2.RouterSpec{
Route: &v1alpha2.GatewayRoutesSpec{
HTTP: &v1alpha2.HTTPRouteSpec{
Spec: &gwapiv1.HTTPRouteSpec{
CommonRouteSpec: gwapiv1.CommonRouteSpec{
ParentRefs: []gwapiv1.ParentReference{
{
Name: "my-parent",
},
},
},
Hostnames: nil,
Rules: nil,
},
Refs: []corev1.LocalObjectReference{{Name: "my-route"}},
},
},
Gateway: &v1alpha2.GatewaySpec{
Refs: []v1alpha2.GatewayObjectReference{{UntypedObjectReference: v1alpha2.UntypedObjectReference{Name: "my-gateway"}}},
},
},
},
{
Router: &v1alpha2.RouterSpec{
Route: &v1alpha2.GatewayRoutesSpec{
HTTP: &v1alpha2.HTTPRouteSpec{
Refs: []corev1.LocalObjectReference{{Name: "my-second-route"}},
},
},
Gateway: &v1alpha2.GatewaySpec{
Refs: []v1alpha2.GatewayObjectReference{{UntypedObjectReference: v1alpha2.UntypedObjectReference{Name: "my-second-gateway"}}},
},
},
},
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Parallelism: &v1alpha2.ParallelismSpec{
Tensor: ptr.To[int32](1),
Pipeline: ptr.To[int32](1),
},
Worker: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "main",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("1"),
"nvidia.com/gpu": resource.MustParse("1"),
},
},
},
},
},
},
},
{
Router: &v1alpha2.RouterSpec{
Scheduler: &v1alpha2.SchedulerSpec{
Pool: &v1alpha2.InferencePoolSpec{
Ref: &corev1.LocalObjectReference{
Name: "my-pool",
},
},
},
},
},
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Parallelism: &v1alpha2.ParallelismSpec{
Tensor: ptr.To[int32](4),
Pipeline: ptr.To[int32](2),
},
Worker: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "main",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("1"),
"nvidia.com/gpu": resource.MustParse("4"),
},
},
},
},
},
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
Router: &v1alpha2.RouterSpec{
Route: &v1alpha2.GatewayRoutesSpec{
HTTP: &v1alpha2.HTTPRouteSpec{
Spec: &gwapiv1.HTTPRouteSpec{
CommonRouteSpec: gwapiv1.CommonRouteSpec{
ParentRefs: []gwapiv1.ParentReference{
{
Name: "my-parent",
},
},
},
Hostnames: nil,
Rules: nil,
},
Refs: []corev1.LocalObjectReference{{Name: "my-second-route"}},
},
},
Gateway: &v1alpha2.GatewaySpec{
Refs: []v1alpha2.GatewayObjectReference{{UntypedObjectReference: v1alpha2.UntypedObjectReference{Name: "my-second-gateway"}}},
},
Scheduler: &v1alpha2.SchedulerSpec{
Pool: &v1alpha2.InferencePoolSpec{
Ref: &corev1.LocalObjectReference{
Name: "my-pool",
},
},
},
},
WorkloadSpec: v1alpha2.WorkloadSpec{
Parallelism: &v1alpha2.ParallelismSpec{
Tensor: ptr.To[int32](4),
Pipeline: ptr.To[int32](2),
},
Worker: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "main",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("1"),
"nvidia.com/gpu": resource.MustParse("4"),
},
},
},
},
},
},
},
},
{
name: "merge requests and limits",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{
Router: &v1alpha2.RouterSpec{
Route: &v1alpha2.GatewayRoutesSpec{},
Gateway: &v1alpha2.GatewaySpec{},
},
},
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Parallelism: &v1alpha2.ParallelismSpec{
Tensor: ptr.To[int32](1),
Pipeline: ptr.To[int32](1),
},
Worker: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "main",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("1"),
"nvidia.com/gpu": resource.MustParse("1"),
},
Limits: corev1.ResourceList{
"nvidia.com/gpu": resource.MustParse("1"),
},
},
Env: []corev1.EnvVar{
{Name: "a", Value: "1"},
{Name: "z", Value: "42"},
},
Args: []string{
"a", "b",
},
},
},
},
},
},
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Worker: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "main",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceMemory: resource.MustParse("1Gi"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("2"),
},
},
Env: []corev1.EnvVar{
{Name: "b", Value: "2"},
{Name: "z", Value: ""},
},
Args: []string{
"x", "y",
},
},
},
},
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
Router: &v1alpha2.RouterSpec{
Route: &v1alpha2.GatewayRoutesSpec{},
Gateway: &v1alpha2.GatewaySpec{},
},
WorkloadSpec: v1alpha2.WorkloadSpec{
Parallelism: &v1alpha2.ParallelismSpec{
Tensor: ptr.To[int32](1),
Pipeline: ptr.To[int32](1),
},
Worker: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "main",
Resources: corev1.ResourceRequirements{
Requests: corev1.ResourceList{
corev1.ResourceMemory: resource.MustParse("1Gi"),
corev1.ResourceCPU: resource.MustParse("1"),
"nvidia.com/gpu": resource.MustParse("1"),
},
Limits: corev1.ResourceList{
corev1.ResourceCPU: resource.MustParse("2"),
"nvidia.com/gpu": resource.MustParse("1"),
},
},
Env: []corev1.EnvVar{
{Name: "b", Value: "2"},
{Name: "a", Value: "1"},
{Name: "z", Value: "42"},
},
Args: []string{
"x", "y",
},
},
},
},
},
},
},
{
name: "merge LoRA adapters",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{
Model: v1alpha2.LLMModelSpec{
URI: apis.URL{Path: "base-model"},
LoRA: &v1alpha2.LoRASpec{
Adapters: []v1alpha2.LLMModelSpec{
{URI: apis.URL{Path: "lora-model"}},
},
},
},
},
{
Model: v1alpha2.LLMModelSpec{
LoRA: &v1alpha2.LoRASpec{
Adapters: []v1alpha2.LLMModelSpec{
{URI: apis.URL{Path: "lora-model2"}},
},
},
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{
URI: apis.URL{Path: "base-model"},
LoRA: &v1alpha2.LoRASpec{
Adapters: []v1alpha2.LLMModelSpec{
{URI: apis.URL{Path: "lora-model2"}},
},
},
},
},
},
{
name: "merge model URI",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{
Model: v1alpha2.LLMModelSpec{
URI: apis.URL{Scheme: "hf", Host: "hub.com", Path: "/model-a"},
},
},
{
Model: v1alpha2.LLMModelSpec{
URI: apis.URL{Scheme: "s3", Host: "bucket.com", Path: "/model-b"},
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{
URI: apis.URL{Scheme: "s3", Host: "bucket.com", Path: "/model-b"},
},
},
},
{
name: "merge baseRefs",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{
BaseRefs: []corev1.LocalObjectReference{
{Name: "base-config-1"},
{Name: "base-config-2"},
},
},
{
BaseRefs: []corev1.LocalObjectReference{
{Name: "override-config-1"},
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
BaseRefs: []corev1.LocalObjectReference{
{Name: "override-config-1"},
},
},
},
{
name: "merge ingress spec",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{
Router: &v1alpha2.RouterSpec{
Ingress: &v1alpha2.IngressSpec{
Refs: []v1alpha2.UntypedObjectReference{
{Name: "base-ingress", Namespace: "base-ns"},
},
},
},
},
{
Router: &v1alpha2.RouterSpec{
Ingress: &v1alpha2.IngressSpec{
Refs: []v1alpha2.UntypedObjectReference{
{Name: "override-ingress", Namespace: "override-ns"},
},
},
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
Router: &v1alpha2.RouterSpec{
Ingress: &v1alpha2.IngressSpec{
Refs: []v1alpha2.UntypedObjectReference{
{Name: "override-ingress", Namespace: "override-ns"},
},
},
},
},
},
{
name: "merge with nil pointer handling",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{
Model: v1alpha2.LLMModelSpec{
URI: apis.URL{Path: "model-uri"},
Name: ptr.To("base-name"),
},
WorkloadSpec: v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](1),
},
},
{
Model: v1alpha2.LLMModelSpec{
Name: nil, // nil pointer should not override non-nil base
},
WorkloadSpec: v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](3),
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{
URI: apis.URL{Path: "model-uri"},
Name: ptr.To("base-name"), // Base value should be preserved
},
WorkloadSpec: v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](3),
},
},
},
{
name: "merge complex nested structures",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{
Model: v1alpha2.LLMModelSpec{
URI: apis.URL{Path: "base-model"},
Name: ptr.To("base-name"),
// Criticality: ptr.To(igwapi.Sheddable),
LoRA: &v1alpha2.LoRASpec{
Adapters: []v1alpha2.LLMModelSpec{
{URI: apis.URL{Path: "lora-model"}},
},
},
},
WorkloadSpec: v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](1),
Parallelism: &v1alpha2.ParallelismSpec{
Tensor: ptr.To[int32](2),
},
},
Router: &v1alpha2.RouterSpec{
Gateway: &v1alpha2.GatewaySpec{
Refs: []v1alpha2.GatewayObjectReference{{UntypedObjectReference: v1alpha2.UntypedObjectReference{Name: "base-gw"}}},
},
},
},
{
Model: v1alpha2.LLMModelSpec{
Name: ptr.To("override-name"),
// Criticality: ptr.To(igwapi.Critical),
LoRA: &v1alpha2.LoRASpec{
Adapters: []v1alpha2.LLMModelSpec{
{URI: apis.URL{Path: "lora-model2"}},
},
},
},
WorkloadSpec: v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](5),
Parallelism: &v1alpha2.ParallelismSpec{
Pipeline: ptr.To[int32](4),
},
},
Router: &v1alpha2.RouterSpec{
Route: &v1alpha2.GatewayRoutesSpec{
HTTP: &v1alpha2.HTTPRouteSpec{
Refs: []corev1.LocalObjectReference{{Name: "override-route"}},
},
},
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{
URI: apis.URL{Path: "base-model"}, // Base URI preserved
Name: ptr.To("override-name"), // Override name
// Criticality: ptr.To(igwapi.Critical),
LoRA: &v1alpha2.LoRASpec{
Adapters: []v1alpha2.LLMModelSpec{
{URI: apis.URL{Path: "lora-model2"}},
},
},
},
WorkloadSpec: v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](5),
Parallelism: &v1alpha2.ParallelismSpec{
Tensor: ptr.To[int32](2), // Base tensor preserved
Pipeline: ptr.To[int32](4), // Override pipeline
},
},
Router: &v1alpha2.RouterSpec{
Gateway: &v1alpha2.GatewaySpec{
Refs: []v1alpha2.GatewayObjectReference{{UntypedObjectReference: v1alpha2.UntypedObjectReference{Name: "base-gw"}}},
},
Route: &v1alpha2.GatewayRoutesSpec{
HTTP: &v1alpha2.HTTPRouteSpec{
Refs: []corev1.LocalObjectReference{{Name: "override-route"}},
},
},
},
},
},
{
name: "merge empty structures",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{
Model: v1alpha2.LLMModelSpec{},
},
{
Model: v1alpha2.LLMModelSpec{
URI: apis.URL{Path: "populated-model"},
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{
URI: apis.URL{Path: "populated-model"},
},
},
},
{
name: "merge with zero values vs nil pointers",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](0), // Zero value, but non-nil pointer
},
},
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Replicas: nil, // Nil pointer should not override zero value
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Replicas: ptr.To[int32](0), // Zero value should be preserved
},
},
},
{
name: "merge pod spec with nil containers",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Image: "busybox",
Name: "busybox",
},
},
InitContainers: []corev1.Container{
{
Image: "busybox-init",
Name: "busybox-init",
},
},
Volumes: []corev1.Volume{
{
Name: "vol1",
},
},
},
Replicas: ptr.To[int32](1),
},
},
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{},
Replicas: nil,
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Image: "busybox",
Name: "busybox",
},
},
InitContainers: []corev1.Container{
{
Image: "busybox-init",
Name: "busybox-init",
},
},
Volumes: []corev1.Volume{
{
Name: "vol1",
},
},
},
Replicas: ptr.To[int32](1),
},
},
},
{
name: "merge pod spec with empty containers",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Image: "busybox",
Name: "busybox",
},
},
InitContainers: []corev1.Container{
{
Image: "busybox-init",
Name: "busybox-init",
},
},
Volumes: []corev1.Volume{
{
Name: "vol1",
},
},
},
Replicas: ptr.To[int32](1),
},
},
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{},
},
Replicas: nil,
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Image: "busybox",
Name: "busybox",
},
},
InitContainers: []corev1.Container{
{
Image: "busybox-init",
Name: "busybox-init",
},
},
Volumes: []corev1.Volume{
{
Name: "vol1",
},
},
},
Replicas: ptr.To[int32](1),
},
},
},
{
name: "merge pod spec, add container",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Image: "busybox",
Name: "busybox",
},
},
InitContainers: []corev1.Container{
{
Image: "busybox-sidecar",
Name: "busybox-sidecar",
},
},
},
Replicas: ptr.To[int32](2),
},
},
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Image: "busybox-2",
Name: "busybox-2",
},
},
InitContainers: []corev1.Container{},
},
Replicas: nil,
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Image: "busybox-2",
Name: "busybox-2",
},
{
Image: "busybox",
Name: "busybox",
},
},
InitContainers: []corev1.Container{
{
Image: "busybox-sidecar",
Name: "busybox-sidecar",
},
},
},
Replicas: ptr.To[int32](2),
},
},
},
{
name: "merge pod spec, add container",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Image: "busybox",
Name: "busybox",
},
},
InitContainers: []corev1.Container{
{
Image: "busybox-sidecar",
Name: "busybox-sidecar",
},
},
},
Replicas: ptr.To[int32](2),
},
},
{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Image: "busybox-2",
Name: "busybox-2",
},
},
InitContainers: []corev1.Container{},
},
Replicas: nil,
},
},
{},
},
want: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Image: "busybox-2",
Name: "busybox-2",
},
{
Image: "busybox",
Name: "busybox",
},
},
InitContainers: []corev1.Container{
{
Image: "busybox-sidecar",
Name: "busybox-sidecar",
},
},
},
Replicas: ptr.To[int32](2),
},
},
},
{
name: "custom pool selector overrides base empty selector",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
// Base config with pool spec but no selector
{
Router: &v1alpha2.RouterSpec{
Route: &v1alpha2.GatewayRoutesSpec{},
Gateway: &v1alpha2.GatewaySpec{},
Scheduler: &v1alpha2.SchedulerSpec{
Pool: &v1alpha2.InferencePoolSpec{
Spec: &igwapi.InferencePoolSpec{
TargetPorts: []igwapi.Port{{Number: 8000}},
},
},
Template: &corev1.PodSpec{
Containers: []corev1.Container{{Name: "main"}},
},
},
},
},
// Override with custom selector for multi-GPU-vendor pooling
{
Router: &v1alpha2.RouterSpec{
Scheduler: &v1alpha2.SchedulerSpec{
Pool: &v1alpha2.InferencePoolSpec{
Spec: &igwapi.InferencePoolSpec{
Selector: igwapi.LabelSelector{
MatchLabels: map[igwapi.LabelKey]igwapi.LabelValue{
"llm-pool": "qwen2-7b",
"kserve.io/component": "workload",
},
},
TargetPorts: []igwapi.Port{{Number: 8000}},
},
},
},
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
Router: &v1alpha2.RouterSpec{
Route: &v1alpha2.GatewayRoutesSpec{},
Gateway: &v1alpha2.GatewaySpec{},
Scheduler: &v1alpha2.SchedulerSpec{
Pool: &v1alpha2.InferencePoolSpec{
Spec: &igwapi.InferencePoolSpec{
Selector: igwapi.LabelSelector{
MatchLabels: map[igwapi.LabelKey]igwapi.LabelValue{
"llm-pool": "qwen2-7b",
"kserve.io/component": "workload",
},
},
TargetPorts: []igwapi.Port{{Number: 8000}},
},
},
Template: &corev1.PodSpec{
Containers: []corev1.Container{{Name: "main"}},
},
},
},
},
},
{
name: "custom pool selector merge base selector",
cfgs: []v1alpha2.LLMInferenceServiceSpec{
// Base config with pool spec but no selector
{
Router: &v1alpha2.RouterSpec{
Route: &v1alpha2.GatewayRoutesSpec{},
Gateway: &v1alpha2.GatewaySpec{},
Scheduler: &v1alpha2.SchedulerSpec{
Pool: &v1alpha2.InferencePoolSpec{
Spec: &igwapi.InferencePoolSpec{
Selector: igwapi.LabelSelector{
MatchLabels: map[igwapi.LabelKey]igwapi.LabelValue{
"x": "x",
},
},
TargetPorts: []igwapi.Port{{Number: 8000}},
},
},
Template: &corev1.PodSpec{
Containers: []corev1.Container{{Name: "main"}},
},
},
},
},
// Override with custom selector for multi-GPU-vendor pooling
{
Router: &v1alpha2.RouterSpec{
Scheduler: &v1alpha2.SchedulerSpec{
Pool: &v1alpha2.InferencePoolSpec{
Spec: &igwapi.InferencePoolSpec{
Selector: igwapi.LabelSelector{
MatchLabels: map[igwapi.LabelKey]igwapi.LabelValue{
"llm-pool": "qwen2-7b",
"kserve.io/component": "workload",
},
},
TargetPorts: []igwapi.Port{{Number: 8000}},
},
},
},
},
},
},
want: v1alpha2.LLMInferenceServiceSpec{
Router: &v1alpha2.RouterSpec{
Route: &v1alpha2.GatewayRoutesSpec{},
Gateway: &v1alpha2.GatewaySpec{},
Scheduler: &v1alpha2.SchedulerSpec{
Pool: &v1alpha2.InferencePoolSpec{
Spec: &igwapi.InferencePoolSpec{
Selector: igwapi.LabelSelector{
MatchLabels: map[igwapi.LabelKey]igwapi.LabelValue{
"llm-pool": "qwen2-7b",
"kserve.io/component": "workload",
"x": "x",
},
},
TargetPorts: []igwapi.Port{{Number: 8000}},
},
},
Template: &corev1.PodSpec{
Containers: []corev1.Container{{Name: "main"}},
},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
ctx := t.Context()
ctx = log.IntoContext(ctx, pkgtest.NewTestLogger(t))
got, err := llmisvc.MergeSpecs(ctx, tt.cfgs...)
if (err != nil) != tt.wantErr {
t.Errorf("MergeSpecs() error = %v, wantErr %v", err, tt.wantErr)
return
}
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Errorf("MergeSpecs() got = \n%#v\n, want \n%#v\nDiff (-want, +got):\n%s", got, tt.want, diff)
}
})
}
}
func TestGetWorkloadLabelSelectorUsedAsDefaultPoolSelector(t *testing.T) {
// Verify that GetWorkloadLabelSelector returns the labels used as default
// InferencePool selector in combineBaseRefsConfig (config_merge.go:232-244).
// When a user provides custom MatchLabels (e.g., for multi-GPU-vendor pooling),
// the default selector is not applied because MatchLabels is non-empty.
meta := metav1.ObjectMeta{Name: "test-svc", Namespace: "default"}
selector := llmisvc.GetWorkloadLabelSelector(meta, nil)
g := NewWithT(t)
g.Expect(selector).To(HaveKeyWithValue("app.kubernetes.io/name", "test-svc"))
g.Expect(selector).To(HaveKeyWithValue("app.kubernetes.io/part-of", "llminferenceservice"))
g.Expect(selector).To(HaveKeyWithValue("kserve.io/component", "workload"))
g.Expect(selector).To(HaveLen(3))
}
func TestReplaceVariables(t *testing.T) {
tests := []struct {
name string
llmSvc *v1alpha2.LLMInferenceService
cfg *v1alpha2.LLMInferenceServiceConfig
extra *llmisvc.Config
want *v1alpha2.LLMInferenceServiceConfig
wantErr bool
}{
{
name: "Replace model name",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{
Name: ptr.To("{{ .Spec.Model.Name }}"),
},
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{
"--served-model-name",
"{{ .Spec.Model.Name }}",
}},
},
},
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{
Spec: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{
Name: ptr.To("meta-llama/Llama-3.2-3B-Instruct"),
},
},
},
want: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{
Name: ptr.To("meta-llama/Llama-3.2-3B-Instruct"),
},
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{
"--served-model-name",
"meta-llama/Llama-3.2-3B-Instruct",
}},
},
},
},
},
},
},
{
name: "template with ChildName function",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
ServiceAccountName: "{{ ChildName .Name `-sa` }}",
Containers: []corev1.Container{
{
Name: "main",
Env: []corev1.EnvVar{
{Name: "DEPLOYMENT_NAME", Value: "{{ ChildName .Name `-deployment` }}"},
},
},
},
},
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{
ObjectMeta: metav1.ObjectMeta{
Name: "test-llm",
Namespace: "test-ns",
},
},
want: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
ServiceAccountName: "test-llm-sa",
Containers: []corev1.Container{
{
Name: "main",
Env: []corev1.EnvVar{
{Name: "DEPLOYMENT_NAME", Value: "test-llm-deployment"},
},
},
},
},
},
},
},
},
{
name: "template in arrays",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "main",
Args: []string{
"--model-name={{ .Name }}",
"--namespace={{ .Namespace }}",
"--config-path=/config/{{ .Name }}.yaml",
},
Env: []corev1.EnvVar{
{Name: "MODEL_NAME", Value: "{{ .Name }}"},
{Name: "NAMESPACE", Value: "{{ .Namespace }}"},
},
},
},
},
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{
ObjectMeta: metav1.ObjectMeta{
Name: "gpt-model",
Namespace: "ml-team",
},
},
want: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "main",
Args: []string{
"--model-name=gpt-model",
"--namespace=ml-team",
"--config-path=/config/gpt-model.yaml",
},
Env: []corev1.EnvVar{
{Name: "MODEL_NAME", Value: "gpt-model"},
{Name: "NAMESPACE", Value: "ml-team"},
},
},
},
},
},
},
},
},
{
name: "template with complex nested model spec",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{
Name: ptr.To("{{ .Spec.Model.Name }}"),
},
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "main",
Args: []string{
"--served-model-name={{ .Spec.Model.Name }}",
"--model-path={{ .Spec.Model.URI.Host }}{{ .Spec.Model.URI.Path }}",
},
},
},
},
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{
Spec: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{
Name: ptr.To("meta-llama/Llama-3.2-3B-Instruct"),
URI: mustParseURL("hf://meta-llama/Llama-3.2-3B-Instruct"),
},
},
},
want: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{
Name: ptr.To("meta-llama/Llama-3.2-3B-Instruct"),
},
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "main",
Args: []string{
"--served-model-name=meta-llama/Llama-3.2-3B-Instruct",
"--model-path=meta-llama/Llama-3.2-3B-Instruct",
},
},
},
},
},
},
},
},
{
name: "template with nil pointer access should not error if default value is provided",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{
Name: ptr.To(`{{ if .Spec.Model.Name }}{{ .Spec.Model.Name }}{{ else }}default-model{{ end }}`),
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{
Spec: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{
Name: nil, // Nil pointer
},
},
},
want: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{
Name: ptr.To("default-model"),
},
},
},
},
{
name: "template with router configurations",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
Router: &v1alpha2.RouterSpec{
Route: &v1alpha2.GatewayRoutesSpec{
HTTP: &v1alpha2.HTTPRouteSpec{
Refs: []corev1.LocalObjectReference{
{Name: "{{ .Name }}-route"},
},
},
},
Gateway: &v1alpha2.GatewaySpec{
Refs: []v1alpha2.GatewayObjectReference{
{UntypedObjectReference: v1alpha2.UntypedObjectReference{Name: "{{ .Name }}-gateway", Namespace: "{{ .Namespace }}"}},
},
},
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{
ObjectMeta: metav1.ObjectMeta{
Name: "router-test",
Namespace: "routing-ns",
},
},
want: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
Router: &v1alpha2.RouterSpec{
Route: &v1alpha2.GatewayRoutesSpec{
HTTP: &v1alpha2.HTTPRouteSpec{
Refs: []corev1.LocalObjectReference{
{Name: "router-test-route"},
},
},
},
Gateway: &v1alpha2.GatewaySpec{
Refs: []v1alpha2.GatewayObjectReference{
{UntypedObjectReference: v1alpha2.UntypedObjectReference{Name: "router-test-gateway", Namespace: "routing-ns"}},
},
},
},
},
},
},
{
name: "template with multiple variables in single string",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "main",
Env: []corev1.EnvVar{
{Name: "FULL_NAME", Value: "{{ .Namespace }}/{{ .Name }}"},
{Name: "CONFIG_PATH", Value: "/config/{{ .Namespace }}-{{ .Name }}.yaml"},
},
},
},
},
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{
ObjectMeta: metav1.ObjectMeta{
Name: "multi-var",
Namespace: "test-ns",
},
},
want: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "main",
Env: []corev1.EnvVar{
{Name: "FULL_NAME", Value: "test-ns/multi-var"},
{Name: "CONFIG_PATH", Value: "/config/test-ns-multi-var.yaml"},
},
},
},
},
},
},
},
},
{
name: "template with invalid syntax should error",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{
Name: ptr.To("{{ .Name"), // Invalid template syntax - missing closing brace
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{
ObjectMeta: metav1.ObjectMeta{Name: "test"},
},
wantErr: true,
},
{
name: "template with non-existent field should error",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
Model: v1alpha2.LLMModelSpec{
Name: ptr.To("{{ .NonExistentField }}"),
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{
ObjectMeta: metav1.ObjectMeta{Name: "test"},
},
wantErr: true,
},
{
name: "template in baseRefs",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
BaseRefs: []corev1.LocalObjectReference{
{Name: "{{ .Name }}-base-config"},
{Name: "{{ .Namespace }}-shared-config"},
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{
ObjectMeta: metav1.ObjectMeta{
Name: "base-ref-test",
Namespace: "template-ns",
},
},
want: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
BaseRefs: []corev1.LocalObjectReference{
{Name: "base-ref-test-base-config"},
{Name: "template-ns-shared-config"},
},
},
},
},
{
name: "shutdownTimeout uses default when spec.template is nil",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{"--shutdown-timeout", "{{ shutdownTimeout .Spec.Template 15 }}"}},
},
},
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{
// spec.template not set — shutdownTimeout falls back to default tgps=60: 60-15-min(5,60)=40
},
want: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{"--shutdown-timeout", "40"}},
},
},
},
},
},
},
{
name: "shutdownTimeout uses spec.template.terminationGracePeriodSeconds when set",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{"--shutdown-timeout", "{{ shutdownTimeout .Spec.Template 15 }}"}},
},
},
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
TerminationGracePeriodSeconds: ptr.To(int64(120)),
},
},
},
},
want: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{"--shutdown-timeout", "100"}},
},
},
},
},
},
},
{
name: "shutdownTimeout uses default when spec.worker is nil",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{"--shutdown-timeout", "{{ shutdownTimeout .Spec.Worker 15 }}"}},
},
},
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{
// spec.worker not set — shutdownTimeout falls back to default tgps=60: 60-15-min(5,60)=40
},
want: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{"--shutdown-timeout", "40"}},
},
},
},
},
},
},
{
name: "shutdownTimeout uses spec.worker.terminationGracePeriodSeconds when set",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{"--shutdown-timeout", "{{ shutdownTimeout .Spec.Worker 15 }}"}},
},
},
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Worker: &corev1.PodSpec{
TerminationGracePeriodSeconds: ptr.To(int64(120)),
},
},
},
},
want: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{"--shutdown-timeout", "100"}},
},
},
},
},
},
},
{
name: "shutdownTimeout uses default when no spec provided (nil arg)",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{"--shutdown-timeout", "{{ shutdownTimeout nil 15 }}"}},
},
},
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{},
want: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{"--shutdown-timeout", "40"}},
},
},
},
},
},
},
{
name: "shutdownTimeout uses spec.prefill.template.terminationGracePeriodSeconds when set",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{"--shutdown-timeout", "{{ if .Spec.Prefill }}{{ shutdownTimeout .Spec.Prefill.Template 15 }}{{ else }}{{ shutdownTimeout nil 15 }}{{ end }}"}},
},
},
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{
Spec: v1alpha2.LLMInferenceServiceSpec{
Prefill: &v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
TerminationGracePeriodSeconds: ptr.To(int64(120)),
},
},
},
},
want: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{"--shutdown-timeout", "100"}},
},
},
},
},
},
},
{
name: "shutdownTimeout uses spec.prefill.worker.terminationGracePeriodSeconds when set",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{"--shutdown-timeout", "{{ if .Spec.Prefill }}{{ shutdownTimeout .Spec.Prefill.Worker 15 }}{{ else }}{{ shutdownTimeout nil 15 }}{{ end }}"}},
},
},
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{
Spec: v1alpha2.LLMInferenceServiceSpec{
Prefill: &v1alpha2.WorkloadSpec{
Worker: &corev1.PodSpec{
TerminationGracePeriodSeconds: ptr.To(int64(120)),
},
},
},
},
want: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{"--shutdown-timeout", "100"}},
},
},
},
},
},
},
{
name: "shutdownTimeout uses default when spec.prefill is nil (prefill via base ref only)",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{"--shutdown-timeout", "{{ if .Spec.Prefill }}{{ shutdownTimeout .Spec.Prefill.Template 15 }}{{ else }}{{ shutdownTimeout nil 15 }}{{ end }}"}},
},
},
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{
Spec: v1alpha2.LLMInferenceServiceSpec{},
},
want: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{"--shutdown-timeout", "40"}},
},
},
},
},
},
},
{
name: "kvTransferConfig returns empty string when kvCacheOffloading is nil",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{"{{ kvTransferConfig .Spec.KVCacheOffloading }}"}},
},
},
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{},
want: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{""}},
},
},
},
},
},
},
{
name: "kvTransferConfig renders --kv-transfer-config from kvCacheOffloading",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{"{{ kvTransferConfig .Spec.KVCacheOffloading }}"}},
},
},
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
KVCacheOffloading: &v1alpha2.KVCacheOffloadingSpec{
CPU: resource.MustParse("10Gi"),
},
},
},
},
want: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
// json.Marshal sorts map keys alphabetically: kv_connector < kv_connector_extra_config < kv_role
// Internally " is escaped to \" for JSON-safe template rendering; JSON unmarshal restores plain ".
{Args: []string{`--kv-transfer-config '{"kv_connector":"OffloadingConnector","kv_connector_extra_config":{"cpu_bytes_to_use":10737418240,"spec_name":"TieringOffloadingSpec"},"kv_role":"kv_both"}'`}},
},
},
},
},
},
},
{
name: "kvTransferConfig includes eviction policy when set",
cfg: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
{Args: []string{"{{ kvTransferConfig .Spec.KVCacheOffloading }}"}},
},
},
},
},
},
llmSvc: &v1alpha2.LLMInferenceService{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
KVCacheOffloading: &v1alpha2.KVCacheOffloadingSpec{
CPU: resource.MustParse("5Gi"),
EvictionPolicy: "arc",
},
},
},
},
want: &v1alpha2.LLMInferenceServiceConfig{
Spec: v1alpha2.LLMInferenceServiceSpec{
WorkloadSpec: v1alpha2.WorkloadSpec{
Template: &corev1.PodSpec{
Containers: []corev1.Container{
// json.Marshal sorts map keys alphabetically: cpu_bytes_to_use < eviction_policy < spec_name
{Args: []string{`--kv-transfer-config '{"kv_connector":"OffloadingConnector","kv_connector_extra_config":{"cpu_bytes_to_use":5368709120,"eviction_policy":"arc","spec_name":"TieringOffloadingSpec"},"kv_role":"kv_both"}'`}},
},
},
},
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := llmisvc.ReplaceVariables(tt.llmSvc, tt.cfg, tt.extra)
if (err != nil) != tt.wantErr {
t.Errorf("ReplaceVariables() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !tt.wantErr {
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Errorf("ReplaceVariables() got = %#v, want %#v\nDiff:\n%s", got, tt.want, diff)
}
}
})
}
}
func mustParseURL(s string) apis.URL {
u, err := apis.ParseURL(s)
if err != nil {
panic(err)
}
return *u
}
func TestAdditionalData(t *testing.T) {
tests := []struct {
name string
config string
reconcilerConfig llmisvc.Config
wantErr bool
want func(llmSvc *v1alpha2.LLMInferenceServiceConfig, g *GomegaWithT)
}{
{
name: "additional structs replacements",
config: `apiVersion: serving.kserve.io/v1alpha1
kind: LLMInferenceServiceConfig
metadata:
name: test-config
namespace: "{{ .GlobalConfig.SystemNamespace }}"
spec:
router:
route:
http:
spec:
parentRefs:
- group: gateway.networking.k8s.io
kind: Gateway
name: "{{ .GlobalConfig.IngressGatewayName }}"
namespace: "{{ .GlobalConfig.IngressGatewayNamespace }}"`,
reconcilerConfig: llmisvc.Config{
SystemNamespace: "my-kserve",
IngressGatewayName: "my-gateway",
IngressGatewayNamespace: "my-ns",
},
want: func(llmSvc *v1alpha2.LLMInferenceServiceConfig, g *GomegaWithT) {
httpRouteSpec := llmSvc.Spec.Router.Route.HTTP.Spec
expectedGatewayRef := gwapiv1.ParentReference{
Name: "my-gateway",
Namespace: ptr.To(gwapiv1.Namespace("my-ns")),
}
g.Expect(httpRouteSpec).To(ktesting.HaveGatewayRefs(expectedGatewayRef))
g.Expect(llmSvc.Namespace).To(Equal("my-kserve"))
},
},
{
name: "template with non-existing key should error",
config: `apiVersion: serving.kserve.io/v1alpha1
kind: LLMInferenceServiceConfig
metadata:
name: "{{ .GlobalConfig.NonExistentConfig.SomeField }}"
spec:
model:
name: "static-model"`,
wantErr: true,
want: func(llmSvc *v1alpha2.LLMInferenceServiceConfig, g *GomegaWithT) {},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
preset := &v1alpha2.LLMInferenceServiceConfig{}
if err := yaml.Unmarshal([]byte(tt.config), preset); err != nil {
t.Errorf("Failed to unmarshal YAML: %v", err)
return
}
llmSvc := &v1alpha2.LLMInferenceService{}
got, err := llmisvc.ReplaceVariables(llmSvc, preset, &tt.reconcilerConfig)
if (err != nil) != tt.wantErr {
t.Errorf("ReplaceVariables() error = %v, wantErr %v", err, tt.wantErr)
return
}
tt.want(got, NewGomegaWithT(t))
})
}
}
func TestToParentRefs(t *testing.T) {
tests := []struct {
name string
refs []v1alpha2.GatewayObjectReference
want []gwapiv1.ParentReference
}{
{
name: "single gateway ref",
refs: []v1alpha2.GatewayObjectReference{
{UntypedObjectReference: v1alpha2.UntypedObjectReference{Name: "my-gateway", Namespace: "my-ns"}},
},
want: []gwapiv1.ParentReference{
{
Name: "my-gateway",
Namespace: ptr.To(gwapiv1.Namespace("my-ns")),
Group: ptr.To(gwapiv1.Group("gateway.networking.k8s.io")),
Kind: ptr.To(gwapiv1.Kind("Gateway")),
},
},
},
{
name: "multiple gateway refs",
refs: []v1alpha2.GatewayObjectReference{
{UntypedObjectReference: v1alpha2.UntypedObjectReference{Name: "gw-1", Namespace: "ns-1"}},
{UntypedObjectReference: v1alpha2.UntypedObjectReference{Name: "gw-2", Namespace: "ns-2"}},
},
want: []gwapiv1.ParentReference{
{
Name: "gw-1",
Namespace: ptr.To(gwapiv1.Namespace("ns-1")),
Group: ptr.To(gwapiv1.Group("gateway.networking.k8s.io")),
Kind: ptr.To(gwapiv1.Kind("Gateway")),
},
{
Name: "gw-2",
Namespace: ptr.To(gwapiv1.Namespace("ns-2")),
Group: ptr.To(gwapiv1.Group("gateway.networking.k8s.io")),
Kind: ptr.To(gwapiv1.Kind("Gateway")),
},
},
},
{
name: "gateway ref with empty namespace",
refs: []v1alpha2.GatewayObjectReference{
{UntypedObjectReference: v1alpha2.UntypedObjectReference{Name: "my-gateway", Namespace: ""}},
},
want: []gwapiv1.ParentReference{
{
Name: "my-gateway",
Namespace: nil,
Group: ptr.To(gwapiv1.Group("gateway.networking.k8s.io")),
Kind: ptr.To(gwapiv1.Kind("Gateway")),
},
},
},
{
name: "empty refs returns empty slice",
refs: []v1alpha2.GatewayObjectReference{},
want: []gwapiv1.ParentReference{},
},
{
name: "nil refs returns empty slice",
refs: nil,
want: []gwapiv1.ParentReference{},
},
{
name: "gateway ref with sectionName targets specific listener",
refs: []v1alpha2.GatewayObjectReference{
{
UntypedObjectReference: v1alpha2.UntypedObjectReference{Name: "shared-gateway", Namespace: "gw-ns"},
SectionName: ptr.To(gwapiv1.SectionName("https")),
},
},
want: []gwapiv1.ParentReference{
{
Name: "shared-gateway",
Namespace: ptr.To(gwapiv1.Namespace("gw-ns")),
Group: ptr.To(gwapiv1.Group("gateway.networking.k8s.io")),
Kind: ptr.To(gwapiv1.Kind("Gateway")),
SectionName: ptr.To(gwapiv1.SectionName("https")),
},
},
},
{
name: "mixed refs - with and without sectionName",
refs: []v1alpha2.GatewayObjectReference{
{UntypedObjectReference: v1alpha2.UntypedObjectReference{Name: "gw-all", Namespace: "ns-1"}},
{
UntypedObjectReference: v1alpha2.UntypedObjectReference{Name: "gw-specific", Namespace: "ns-2"},
SectionName: ptr.To(gwapiv1.SectionName("http")),
},
},
want: []gwapiv1.ParentReference{
{
Name: "gw-all",
Namespace: ptr.To(gwapiv1.Namespace("ns-1")),
Group: ptr.To(gwapiv1.Group("gateway.networking.k8s.io")),
Kind: ptr.To(gwapiv1.Kind("Gateway")),
SectionName: nil,
},
{
Name: "gw-specific",
Namespace: ptr.To(gwapiv1.Namespace("ns-2")),
Group: ptr.To(gwapiv1.Group("gateway.networking.k8s.io")),
Kind: ptr.To(gwapiv1.Kind("Gateway")),
SectionName: ptr.To(gwapiv1.SectionName("http")),
},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := llmisvc.ToParentRefs(tt.refs)
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Errorf("ToParentRefs() mismatch (-want +got):\n%s", diff)
}
})
}
}
func TestParentRefRewritingGuard(t *testing.T) {
tests := []struct {
name string
gatewayHasRefs bool
httpRouteHasSpec bool
wantRewrite bool
}{
{
name: "gateway refs + route spec -> rewrite",
gatewayHasRefs: true,
httpRouteHasSpec: true,
wantRewrite: true,
},
{
name: "gateway refs + route refs (no spec) -> skip rewrite",
gatewayHasRefs: true,
httpRouteHasSpec: false,
wantRewrite: false,
},
{
name: "no gateway refs + route spec -> skip rewrite",
gatewayHasRefs: false,
httpRouteHasSpec: true,
wantRewrite: false,
},
{
name: "no gateway refs + no route spec -> skip rewrite",
gatewayHasRefs: false,
httpRouteHasSpec: false,
wantRewrite: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gateway := &v1alpha2.GatewaySpec{}
if tt.gatewayHasRefs {
gateway.Refs = []v1alpha2.GatewayObjectReference{
{UntypedObjectReference: v1alpha2.UntypedObjectReference{Name: "my-gw", Namespace: "my-ns"}},
}
}
httpRoute := &v1alpha2.HTTPRouteSpec{}
if tt.httpRouteHasSpec {
httpRoute.Spec = &gwapiv1.HTTPRouteSpec{
CommonRouteSpec: gwapiv1.CommonRouteSpec{
ParentRefs: []gwapiv1.ParentReference{
{Name: "old-default-gateway"},
},
},
}
} else {
httpRoute.Refs = []corev1.LocalObjectReference{{Name: "my-route"}}
}
shouldRewrite := httpRoute.HasSpec() && gateway.HasRefs()
if shouldRewrite != tt.wantRewrite {
t.Errorf("expected rewrite=%v, got %v", tt.wantRewrite, shouldRewrite)
}
if shouldRewrite {
parentRefs := llmisvc.ToParentRefs(gateway.Refs)
if len(parentRefs) != len(gateway.Refs) {
t.Errorf("expected %d parent refs, got %d", len(gateway.Refs), len(parentRefs))
}
}
})
}
}
func TestReplaceVariables_TemplateInjection(t *testing.T) {
tests := []struct {
name string
config string
}{
{
name: "access CredentialConfig via template injection",
config: `apiVersion: serving.kserve.io/v1alpha1
kind: LLMInferenceServiceConfig
metadata:
name: "{{ .GlobalConfig.CredentialConfig.StorageSpecSecretName }}"
spec:
model:
name: "safe-model"`,
},
{
name: "access StorageConfig via template injection",
config: `apiVersion: serving.kserve.io/v1alpha1
kind: LLMInferenceServiceConfig
metadata:
name: injected
spec:
model:
name: "{{ .GlobalConfig.StorageConfig }}"`,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
preset := &v1alpha2.LLMInferenceServiceConfig{}
if err := yaml.Unmarshal([]byte(tt.config), preset); err != nil {
t.Fatalf("Failed to unmarshal YAML: %v", err)
}
reconcilerConfig := &llmisvc.Config{
SystemNamespace: "kserve",
IngressGatewayName: "kserve-gateway",
IngressGatewayNamespace: "kserve",
CredentialConfig: &credentials.CredentialConfig{StorageSpecSecretName: "super-secret"},
}
llmSvc := &v1alpha2.LLMInferenceService{}
_, err := llmisvc.ReplaceVariables(llmSvc, preset, reconcilerConfig)
if err == nil {
t.Errorf("ReplaceVariables() should reject template accessing sensitive GlobalConfig fields, but succeeded")
}
})
}
}
func TestWellKnownConfigResolver_Attach(t *testing.T) {
tests := []struct {
name string
disableVersioning bool
llmSvc *v1alpha2.LLMInferenceService
wantAnnotations func(t *testing.T, annotations map[string]string)
}{
{
name: "versioning disabled is a no-op",
disableVersioning: true,
llmSvc: &v1alpha2.LLMInferenceService{
Status: v1alpha2.LLMInferenceServiceStatus{},
},
wantAnnotations: func(t *testing.T, annotations map[string]string) {
if annotations != nil {
t.Errorf("expected nil annotations when versioning disabled, got %v", annotations)
}
},
},
{
name: "pins all well-known configs on first call",
llmSvc: &v1alpha2.LLMInferenceService{
Status: v1alpha2.LLMInferenceServiceStatus{},
},
wantAnnotations: func(t *testing.T, annotations map[string]string) {
if annotations == nil {
t.Fatal("expected annotations to be set")
}
for _, name := range llmisvc.WellKnownDefaultConfigs.UnsortedList() {
suffix := strings.TrimPrefix(name, "kserve-")
key := llmisvc.StaticWellKnownConfigResolverPrefix + suffix
if got, ok := annotations[key]; !ok {
t.Errorf("missing annotation key %q", key)
} else if got != name {
t.Errorf("annotation %q = %q, want %q", key, got, name)
}
}
},
},
{
name: "idempotent: calling Attach twice does not change annotations",
llmSvc: &v1alpha2.LLMInferenceService{
Status: v1alpha2.LLMInferenceServiceStatus{},
},
wantAnnotations: func(t *testing.T, annotations map[string]string) {
// This test calls Attach twice (handled in the test body below);
// the check ensures values didn't change.
if annotations == nil {
t.Fatal("expected annotations to be set")
}
for _, name := range llmisvc.WellKnownDefaultConfigs.UnsortedList() {
suffix := strings.TrimPrefix(name, "kserve-")
key := llmisvc.StaticWellKnownConfigResolverPrefix + suffix
if got := annotations[key]; got != name {
t.Errorf("annotation %q = %q, want %q after second Attach", key, got, name)
}
}
},
},
{
name: "preserves pre-populated annotations",
llmSvc: &v1alpha2.LLMInferenceService{
Status: v1alpha2.LLMInferenceServiceStatus{
Status: duckv1.Status{
Annotations: map[string]string{
llmisvc.StaticWellKnownConfigResolverPrefix + "config-llm-template": "custom-prefix-config-llm-template",
"unrelated-key": "unrelated-value",
},
},
},
},
wantAnnotations: func(t *testing.T, annotations map[string]string) {
if annotations == nil {
t.Fatal("expected annotations to be set")
}
// The pre-populated annotation should be preserved, not overwritten.
if got := annotations[llmisvc.StaticWellKnownConfigResolverPrefix+"config-llm-template"]; got != "custom-prefix-config-llm-template" {
t.Errorf("pre-populated annotation was overwritten: got %q, want %q", got, "custom-prefix-config-llm-template")
}
// Unrelated annotations should be preserved.
if got := annotations["unrelated-key"]; got != "unrelated-value" {
t.Errorf("unrelated annotation was lost: got %q, want %q", got, "unrelated-value")
}
// Other well-known configs should still be populated.
suffix := "config-llm-scheduler"
key := llmisvc.StaticWellKnownConfigResolverPrefix + suffix
if got, ok := annotations[key]; !ok {
t.Errorf("missing annotation key %q", key)
} else if got != "kserve-"+suffix {
t.Errorf("annotation %q = %q, want %q", key, got, "kserve-"+suffix)
}
},
},
{
name: "annotations pinned with old prefix survive new Attach",
llmSvc: &v1alpha2.LLMInferenceService{
Status: v1alpha2.LLMInferenceServiceStatus{
Status: duckv1.Status{
Annotations: func() map[string]string {
// Simulate annotations pinned with an old prefix (e.g., "old-prefix-").
m := map[string]string{}
for _, name := range llmisvc.WellKnownDefaultConfigs.UnsortedList() {
suffix := strings.TrimPrefix(name, "kserve-")
key := llmisvc.StaticWellKnownConfigResolverPrefix + suffix
m[key] = "old-prefix-" + suffix
}
return m
}(),
},
},
},
wantAnnotations: func(t *testing.T, annotations map[string]string) {
if annotations == nil {
t.Fatal("expected annotations to be set")
}
// All annotations should retain the old prefix values.
for _, name := range llmisvc.WellKnownDefaultConfigs.UnsortedList() {
suffix := strings.TrimPrefix(name, "kserve-")
key := llmisvc.StaticWellKnownConfigResolverPrefix + suffix
want := "old-prefix-" + suffix
if got := annotations[key]; got != want {
t.Errorf("annotation %q = %q, want %q (old-prefix value should survive)", key, got, want)
}
}
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.disableVersioning {
cleanup := llmisvc.SetUseVersionedConfigForTest(false)
defer cleanup()
}
wr := &llmisvc.WellKnownConfigResolver{}
wr.Attach(tt.llmSvc)
// For the idempotency test, call Attach a second time.
if tt.name == "idempotent: calling Attach twice does not change annotations" {
// Snapshot after first call.
firstCallAnnotations := make(map[string]string, len(tt.llmSvc.Status.Annotations))
for k, v := range tt.llmSvc.Status.Annotations {
firstCallAnnotations[k] = v
}
wr.Attach(tt.llmSvc)
// Verify annotations didn't change.
if diff := cmp.Diff(firstCallAnnotations, tt.llmSvc.Status.Annotations); diff != "" {
t.Errorf("Attach() second call changed annotations (-first +second):\n%s", diff)
}
}
tt.wantAnnotations(t, tt.llmSvc.Status.Annotations)
})
}
}
func TestWellKnownConfigResolver_Resolve(t *testing.T) {
tests := []struct {
name string
disableVersioning bool
llmSvc *v1alpha2.LLMInferenceService
inputName string
want string
}{
{
name: "versioning disabled returns name as-is",
disableVersioning: true,
llmSvc: &v1alpha2.LLMInferenceService{
Status: v1alpha2.LLMInferenceServiceStatus{
Status: duckv1.Status{
Annotations: map[string]string{
llmisvc.StaticWellKnownConfigResolverPrefix + "config-llm-template": "kserve-config-llm-template",
},
},
},
},
inputName: "kserve-config-llm-template",
want: "kserve-config-llm-template",
},
{
name: "nil annotations returns name as-is",
llmSvc: &v1alpha2.LLMInferenceService{
Status: v1alpha2.LLMInferenceServiceStatus{},
},
inputName: "kserve-config-llm-template",
want: "kserve-config-llm-template",
},
{
name: "returns pinned name from annotations",
llmSvc: &v1alpha2.LLMInferenceService{
Status: v1alpha2.LLMInferenceServiceStatus{
Status: duckv1.Status{
Annotations: map[string]string{
llmisvc.StaticWellKnownConfigResolverPrefix + "config-llm-template": "kserve-config-llm-template",
},
},
},
},
inputName: "kserve-config-llm-template",
want: "kserve-config-llm-template",
},
{
name: "returns pinned name even when current prefix differs",
llmSvc: &v1alpha2.LLMInferenceService{
Status: v1alpha2.LLMInferenceServiceStatus{
Status: duckv1.Status{
Annotations: map[string]string{
llmisvc.StaticWellKnownConfigResolverPrefix + "config-llm-template": "old-prefix-config-llm-template",
},
},
},
},
inputName: "kserve-config-llm-template",
want: "old-prefix-config-llm-template",
},
{
name: "suffix not found in annotations returns name as-is",
llmSvc: &v1alpha2.LLMInferenceService{
Status: v1alpha2.LLMInferenceServiceStatus{
Status: duckv1.Status{
Annotations: map[string]string{
llmisvc.StaticWellKnownConfigResolverPrefix + "config-llm-scheduler": "kserve-config-llm-scheduler",
},
},
},
},
inputName: "kserve-config-llm-template",
want: "kserve-config-llm-template",
},
{
name: "name without known prefix strips nothing and looks up full suffix",
llmSvc: &v1alpha2.LLMInferenceService{
Status: v1alpha2.LLMInferenceServiceStatus{
Status: duckv1.Status{
Annotations: map[string]string{
llmisvc.StaticWellKnownConfigResolverPrefix + "custom-config": "pinned-custom-config",
},
},
},
},
inputName: "custom-config",
want: "pinned-custom-config",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.disableVersioning {
cleanup := llmisvc.SetUseVersionedConfigForTest(false)
defer cleanup()
}
wr := &llmisvc.WellKnownConfigResolver{}
got := wr.Resolve(tt.llmSvc, tt.inputName)
if got != tt.want {
t.Errorf("Resolve(%q) = %q, want %q", tt.inputName, got, tt.want)
}
})
}
}
const tlsTemplateFixture = `apiVersion: serving.kserve.io/v1alpha1
kind: LLMInferenceServiceConfig
metadata:
name: test
spec:
router:
scheduler:
template:
containers:
- name: main
args:
- '{{ if .GlobalConfig.EnableTLS }}--enable-cert-reload=true{{- end }}'
- '{{ if .GlobalConfig.EnableTLS }}--secure-serving=true{{- end }}'
- '{{ if .GlobalConfig.EnableTLS }}--model-server-metrics-scheme=https{{- end }}'
- '{{ if .GlobalConfig.EnableTLS }}--cert-path=/var/run/kserve/tls{{- end }}'
`
func TestReplaceVariables_TLSConditional(t *testing.T) {
tests := []struct {
name string
enableTLS bool
wantArgs []string
}{
{
name: "TLS on: all flags render with TLS values",
enableTLS: true,
wantArgs: []string{
"--enable-cert-reload=true",
"--secure-serving=true",
"--model-server-metrics-scheme=https",
"--cert-path=/var/run/kserve/tls",
},
},
{
name: "TLS off: all flags render empty",
enableTLS: false,
wantArgs: []string{
"",
"",
"",
"",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
preset := &v1alpha2.LLMInferenceServiceConfig{}
if err := yaml.Unmarshal([]byte(tlsTemplateFixture), preset); err != nil {
t.Fatalf("failed to unmarshal fixture: %v", err)
}
llmSvc := &v1alpha2.LLMInferenceService{}
cfg := &llmisvc.Config{EnableTLS: tt.enableTLS}
got, err := llmisvc.ReplaceVariables(llmSvc, preset, cfg)
if err != nil {
t.Fatalf("ReplaceVariables() error = %v", err)
}
containers := got.Spec.Router.Scheduler.Template.Containers
if len(containers) == 0 {
t.Fatal("expected at least one container in rendered config")
}
args := containers[0].Args
if len(args) != len(tt.wantArgs) {
t.Fatalf("got %d args, want %d: %q", len(args), len(tt.wantArgs), args)
}
for i, want := range tt.wantArgs {
if args[i] != want {
t.Errorf("arg[%d] = %q, want %q", i, args[i], want)
}
}
})
}
}
func TestReplaceVariables_PDSidecarTLSConditional(t *testing.T) {
const pdSidecarFixture = `apiVersion: serving.kserve.io/v1alpha1
kind: LLMInferenceServiceConfig
metadata:
name: test
spec:
router:
scheduler:
template:
containers:
- name: sidecar
args:
- '{{ if .GlobalConfig.EnableTLS }}--decoder-use-tls=true{{- end }}'
- '{{ if .GlobalConfig.EnableTLS }}--prefiller-use-tls=true{{- end }}'
`
tests := []struct {
name string
enableTLS bool
wantArgs []string
}{
{
name: "TLS on",
enableTLS: true,
wantArgs: []string{"--decoder-use-tls=true", "--prefiller-use-tls=true"},
},
{
name: "TLS off",
enableTLS: false,
wantArgs: []string{"", ""},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
preset := &v1alpha2.LLMInferenceServiceConfig{}
if err := yaml.Unmarshal([]byte(pdSidecarFixture), preset); err != nil {
t.Fatalf("unmarshal: %v", err)
}
got, err := llmisvc.ReplaceVariables(&v1alpha2.LLMInferenceService{}, preset, &llmisvc.Config{EnableTLS: tt.enableTLS})
if err != nil {
t.Fatalf("ReplaceVariables: %v", err)
}
args := got.Spec.Router.Scheduler.Template.Containers[0].Args
if len(args) != len(tt.wantArgs) {
t.Fatalf("got %d args, want %d: %q", len(args), len(tt.wantArgs), args)
}
for i, want := range tt.wantArgs {
if args[i] != want {
t.Errorf("arg[%d] = %q, want %q", i, args[i], want)
}
}
})
}
}
Reference in New Issue View Git Blame Copy Permalink