mirror of
https://github.com/hanzo-ml/kserve.git
synced 2026-10-30 20:53:49 +00:00
637 lines
18 KiB
Go
637 lines
18 KiB
Go
/*
|
|
Copyright 2025 The KServe Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package llmisvc_test
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/google/go-cmp/cmp"
|
|
corev1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/equality"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/util/intstr"
|
|
"k8s.io/utils/ptr"
|
|
"sigs.k8s.io/yaml"
|
|
|
|
"github.com/kserve/kserve/pkg/controller/v1alpha2/llmisvc"
|
|
|
|
kservetesting "github.com/kserve/kserve/pkg/testing"
|
|
|
|
"github.com/kserve/kserve/pkg/apis/serving/v1alpha2"
|
|
)
|
|
|
|
// TODO(webhook): re-use webhook logic to do the spec merge and validation
|
|
func TestPresetFiles(t *testing.T) {
|
|
presetsDir := filepath.Join(kservetesting.ProjectRoot(), "config", "llmisvcconfig")
|
|
|
|
llmSvc := llmisvc.LLMInferenceServiceSample()
|
|
kserveSystemConfig := llmisvc.Config{
|
|
SystemNamespace: "kserve",
|
|
IngressGatewayName: "kserve-ingress-gateway",
|
|
IngressGatewayNamespace: "kserve",
|
|
}
|
|
|
|
tt := map[string]struct {
|
|
expected *v1alpha2.LLMInferenceServiceConfig
|
|
}{
|
|
"config-llm-decode-worker-data-parallel.yaml": {
|
|
expected: &v1alpha2.LLMInferenceServiceConfig{
|
|
TypeMeta: metav1.TypeMeta{
|
|
APIVersion: "serving.kserve.io/v1alpha2",
|
|
Kind: "LLMInferenceServiceConfig",
|
|
},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "kserve-config-llm-decode-worker-data-parallel",
|
|
},
|
|
Spec: v1alpha2.LLMInferenceServiceSpec{
|
|
WorkloadSpec: v1alpha2.WorkloadSpec{
|
|
Template: &corev1.PodSpec{
|
|
Volumes: []corev1.Volume{
|
|
{
|
|
Name: "home",
|
|
VolumeSource: corev1.VolumeSource{
|
|
EmptyDir: &corev1.EmptyDirVolumeSource{},
|
|
},
|
|
},
|
|
{
|
|
Name: "tmp-dir",
|
|
VolumeSource: corev1.VolumeSource{
|
|
EmptyDir: &corev1.EmptyDirVolumeSource{},
|
|
},
|
|
},
|
|
{
|
|
Name: "dshm",
|
|
VolumeSource: corev1.VolumeSource{
|
|
EmptyDir: &corev1.EmptyDirVolumeSource{
|
|
Medium: corev1.StorageMediumMemory,
|
|
SizeLimit: ptr.To(resource.MustParse("8Gi")),
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "model-cache",
|
|
VolumeSource: corev1.VolumeSource{
|
|
EmptyDir: &corev1.EmptyDirVolumeSource{},
|
|
},
|
|
},
|
|
{
|
|
Name: "tls-certs",
|
|
VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "test-llm-preset-kserve-self-signed-certs"}},
|
|
},
|
|
},
|
|
InitContainers: []corev1.Container{
|
|
{
|
|
Name: "llm-d-routing-sidecar",
|
|
Image: "ghcr.io/llm-d/llm-d-routing-sidecar:v0.7.1",
|
|
Command: []string{
|
|
"/app/pd-sidecar",
|
|
"--port=8000",
|
|
"--vllm-port=8001",
|
|
"--connector=nixlv2",
|
|
"--enable-ssrf-protection=true",
|
|
"--pool-group=inference.networking.x-k8s.io",
|
|
"--secure-proxy=false",
|
|
"",
|
|
"",
|
|
"",
|
|
},
|
|
Env: []corev1.EnvVar{
|
|
{
|
|
Name: "INFERENCE_POOL_NAMESPACE",
|
|
ValueFrom: &corev1.EnvVarSource{
|
|
FieldRef: &corev1.ObjectFieldSelector{
|
|
FieldPath: "metadata.namespace",
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "SSL_CERT_DIR",
|
|
Value: "/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs",
|
|
},
|
|
},
|
|
Ports: []corev1.ContainerPort{
|
|
{
|
|
ContainerPort: 8000,
|
|
Protocol: corev1.ProtocolTCP,
|
|
},
|
|
},
|
|
SecurityContext: &corev1.SecurityContext{
|
|
AllowPrivilegeEscalation: ptr.To(false),
|
|
RunAsNonRoot: ptr.To(true),
|
|
Capabilities: &corev1.Capabilities{
|
|
Drop: []corev1.Capability{"ALL"},
|
|
},
|
|
ReadOnlyRootFilesystem: ptr.To(true),
|
|
SeccompProfile: &corev1.SeccompProfile{
|
|
Type: corev1.SeccompProfileTypeRuntimeDefault,
|
|
},
|
|
},
|
|
RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways),
|
|
TerminationMessagePath: "/dev/termination-log",
|
|
TerminationMessagePolicy: "FallbackToLogsOnError",
|
|
ImagePullPolicy: "IfNotPresent",
|
|
VolumeMounts: []corev1.VolumeMount{
|
|
{
|
|
Name: "tls-certs",
|
|
ReadOnly: true,
|
|
MountPath: "/var/run/kserve/tls",
|
|
},
|
|
},
|
|
ReadinessProbe: &corev1.Probe{
|
|
ProbeHandler: corev1.ProbeHandler{
|
|
HTTPGet: &corev1.HTTPGetAction{
|
|
Path: "/health",
|
|
Port: intstr.FromInt32(8000),
|
|
Scheme: corev1.URISchemeHTTP,
|
|
},
|
|
},
|
|
InitialDelaySeconds: 10,
|
|
TimeoutSeconds: 5,
|
|
PeriodSeconds: 10,
|
|
FailureThreshold: 10,
|
|
},
|
|
LivenessProbe: &corev1.Probe{
|
|
ProbeHandler: corev1.ProbeHandler{
|
|
HTTPGet: &corev1.HTTPGetAction{
|
|
Path: "/health",
|
|
Port: intstr.FromInt32(8000),
|
|
Scheme: corev1.URISchemeHTTP,
|
|
},
|
|
},
|
|
InitialDelaySeconds: 10,
|
|
TimeoutSeconds: 10,
|
|
PeriodSeconds: 10,
|
|
FailureThreshold: 3,
|
|
},
|
|
},
|
|
},
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "main",
|
|
Image: "ghcr.io/llm-d/llm-d-cuda:v0.6.0",
|
|
Ports: []corev1.ContainerPort{
|
|
{
|
|
ContainerPort: 8001,
|
|
Protocol: corev1.ProtocolTCP,
|
|
},
|
|
},
|
|
Env: []corev1.EnvVar{
|
|
{
|
|
Name: "HOME",
|
|
Value: "/home",
|
|
},
|
|
{
|
|
Name: "VLLM_LOGGING_LEVEL",
|
|
Value: "INFO",
|
|
},
|
|
{
|
|
Name: "HF_HUB_CACHE",
|
|
Value: "/models",
|
|
},
|
|
},
|
|
VolumeMounts: []corev1.VolumeMount{
|
|
{
|
|
Name: "home",
|
|
MountPath: "/home",
|
|
},
|
|
{
|
|
Name: "tmp-dir",
|
|
MountPath: "/tmp",
|
|
},
|
|
{
|
|
Name: "dshm",
|
|
MountPath: "/dev/shm",
|
|
},
|
|
{
|
|
Name: "model-cache",
|
|
MountPath: "/models",
|
|
},
|
|
{
|
|
Name: "tls-certs",
|
|
ReadOnly: true,
|
|
MountPath: "/var/run/kserve/tls",
|
|
},
|
|
},
|
|
LivenessProbe: &corev1.Probe{
|
|
ProbeHandler: corev1.ProbeHandler{
|
|
HTTPGet: &corev1.HTTPGetAction{
|
|
Path: "/health",
|
|
Port: intstr.FromInt32(8001),
|
|
Scheme: corev1.URISchemeHTTP,
|
|
},
|
|
},
|
|
|
|
TimeoutSeconds: 10,
|
|
PeriodSeconds: 10,
|
|
FailureThreshold: 3,
|
|
},
|
|
ReadinessProbe: &corev1.Probe{
|
|
ProbeHandler: corev1.ProbeHandler{
|
|
HTTPGet: &corev1.HTTPGetAction{
|
|
Path: "/health",
|
|
Port: intstr.FromInt32(8001),
|
|
Scheme: corev1.URISchemeHTTP,
|
|
},
|
|
},
|
|
|
|
TimeoutSeconds: 5,
|
|
PeriodSeconds: 30,
|
|
FailureThreshold: 60,
|
|
},
|
|
StartupProbe: &corev1.Probe{
|
|
ProbeHandler: corev1.ProbeHandler{
|
|
HTTPGet: &corev1.HTTPGetAction{
|
|
Path: "/health",
|
|
Port: intstr.FromInt32(8001),
|
|
Scheme: corev1.URISchemeHTTP,
|
|
},
|
|
},
|
|
FailureThreshold: 60,
|
|
PeriodSeconds: 10,
|
|
},
|
|
TerminationMessagePath: "/dev/termination-log",
|
|
TerminationMessagePolicy: "FallbackToLogsOnError",
|
|
ImagePullPolicy: "IfNotPresent",
|
|
SecurityContext: &corev1.SecurityContext{
|
|
Capabilities: &corev1.Capabilities{
|
|
Add: []corev1.Capability{
|
|
"IPC_LOCK",
|
|
"SYS_RAWIO",
|
|
"NET_RAW",
|
|
},
|
|
Drop: []corev1.Capability{"ALL"},
|
|
},
|
|
AllowPrivilegeEscalation: ptr.To(false),
|
|
RunAsNonRoot: ptr.To(true),
|
|
ReadOnlyRootFilesystem: ptr.To(true),
|
|
SeccompProfile: &corev1.SeccompProfile{
|
|
Type: corev1.SeccompProfileTypeRuntimeDefault,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
TerminationGracePeriodSeconds: ptr.To(int64(30)),
|
|
},
|
|
Worker: &corev1.PodSpec{
|
|
Volumes: []corev1.Volume{
|
|
{
|
|
Name: "home",
|
|
VolumeSource: corev1.VolumeSource{
|
|
EmptyDir: &corev1.EmptyDirVolumeSource{},
|
|
},
|
|
},
|
|
{
|
|
Name: "tmp-dir",
|
|
VolumeSource: corev1.VolumeSource{
|
|
EmptyDir: &corev1.EmptyDirVolumeSource{},
|
|
},
|
|
},
|
|
{
|
|
Name: "dshm",
|
|
VolumeSource: corev1.VolumeSource{
|
|
EmptyDir: &corev1.EmptyDirVolumeSource{
|
|
Medium: corev1.StorageMediumMemory,
|
|
SizeLimit: ptr.To(resource.MustParse("8Gi")),
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "model-cache",
|
|
VolumeSource: corev1.VolumeSource{
|
|
EmptyDir: &corev1.EmptyDirVolumeSource{},
|
|
},
|
|
},
|
|
{
|
|
Name: "tls-certs",
|
|
VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "test-llm-preset-kserve-self-signed-certs"}},
|
|
},
|
|
},
|
|
TerminationGracePeriodSeconds: ptr.To(int64(30)),
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "main",
|
|
Image: "ghcr.io/llm-d/llm-d-cuda:v0.6.0",
|
|
Ports: []corev1.ContainerPort{
|
|
{
|
|
ContainerPort: 8001,
|
|
Protocol: corev1.ProtocolTCP,
|
|
},
|
|
},
|
|
VolumeMounts: []corev1.VolumeMount{
|
|
{
|
|
Name: "home",
|
|
MountPath: "/home",
|
|
},
|
|
{
|
|
Name: "tmp-dir",
|
|
MountPath: "/tmp",
|
|
},
|
|
{
|
|
Name: "dshm",
|
|
MountPath: "/dev/shm",
|
|
},
|
|
{
|
|
Name: "model-cache",
|
|
MountPath: "/models",
|
|
},
|
|
{
|
|
Name: "tls-certs",
|
|
ReadOnly: true,
|
|
MountPath: "/var/run/kserve/tls",
|
|
},
|
|
},
|
|
SecurityContext: &corev1.SecurityContext{
|
|
Capabilities: &corev1.Capabilities{
|
|
Add: []corev1.Capability{
|
|
"IPC_LOCK",
|
|
"SYS_RAWIO",
|
|
"NET_RAW",
|
|
},
|
|
Drop: []corev1.Capability{"ALL"},
|
|
},
|
|
AllowPrivilegeEscalation: ptr.To(false),
|
|
RunAsNonRoot: ptr.To(true),
|
|
ReadOnlyRootFilesystem: ptr.To(true),
|
|
SeccompProfile: &corev1.SeccompProfile{
|
|
Type: corev1.SeccompProfileTypeRuntimeDefault,
|
|
},
|
|
},
|
|
Env: []corev1.EnvVar{
|
|
{
|
|
Name: "HOME",
|
|
Value: "/home",
|
|
},
|
|
{
|
|
Name: "VLLM_LOGGING_LEVEL",
|
|
Value: "INFO",
|
|
},
|
|
{
|
|
Name: "HF_HUB_CACHE",
|
|
Value: "/models",
|
|
},
|
|
{
|
|
Name: "VLLM_RANDOMIZE_DP_DUMMY_INPUTS",
|
|
Value: "1",
|
|
},
|
|
},
|
|
TerminationMessagePath: "/dev/termination-log",
|
|
TerminationMessagePolicy: "FallbackToLogsOnError",
|
|
ImagePullPolicy: "IfNotPresent",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
"config-llm-template.yaml": {
|
|
expected: &v1alpha2.LLMInferenceServiceConfig{
|
|
TypeMeta: metav1.TypeMeta{
|
|
APIVersion: "serving.kserve.io/v1alpha2",
|
|
Kind: "LLMInferenceServiceConfig",
|
|
},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "kserve-config-llm-template",
|
|
},
|
|
Spec: v1alpha2.LLMInferenceServiceSpec{
|
|
WorkloadSpec: v1alpha2.WorkloadSpec{
|
|
Template: &corev1.PodSpec{
|
|
Volumes: []corev1.Volume{
|
|
{
|
|
Name: "home",
|
|
VolumeSource: corev1.VolumeSource{
|
|
EmptyDir: &corev1.EmptyDirVolumeSource{},
|
|
},
|
|
},
|
|
{
|
|
Name: "dshm",
|
|
VolumeSource: corev1.VolumeSource{
|
|
EmptyDir: &corev1.EmptyDirVolumeSource{
|
|
Medium: corev1.StorageMediumMemory,
|
|
SizeLimit: ptr.To(resource.MustParse("1Gi")),
|
|
},
|
|
},
|
|
},
|
|
{
|
|
Name: "model-cache",
|
|
VolumeSource: corev1.VolumeSource{
|
|
EmptyDir: &corev1.EmptyDirVolumeSource{},
|
|
},
|
|
},
|
|
{
|
|
Name: "tmp-dir",
|
|
VolumeSource: corev1.VolumeSource{
|
|
EmptyDir: &corev1.EmptyDirVolumeSource{},
|
|
},
|
|
},
|
|
{
|
|
Name: "tls-certs",
|
|
VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "test-llm-preset-kserve-self-signed-certs"}},
|
|
},
|
|
},
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "main",
|
|
Image: "ghcr.io/llm-d/llm-d-cuda:v0.6.0",
|
|
Ports: []corev1.ContainerPort{
|
|
{
|
|
ContainerPort: 8000,
|
|
Protocol: corev1.ProtocolTCP,
|
|
},
|
|
},
|
|
Env: []corev1.EnvVar{
|
|
{
|
|
Name: "HOME",
|
|
Value: "/home",
|
|
},
|
|
{
|
|
Name: "VLLM_LOGGING_LEVEL",
|
|
Value: "INFO",
|
|
},
|
|
{
|
|
Name: "HF_HUB_CACHE",
|
|
Value: "/models",
|
|
},
|
|
},
|
|
VolumeMounts: []corev1.VolumeMount{
|
|
{
|
|
Name: "home",
|
|
MountPath: "/home",
|
|
},
|
|
{
|
|
Name: "tmp-dir",
|
|
MountPath: "/tmp",
|
|
},
|
|
{
|
|
Name: "dshm",
|
|
MountPath: "/dev/shm",
|
|
},
|
|
{
|
|
Name: "model-cache",
|
|
MountPath: "/models",
|
|
},
|
|
{
|
|
Name: "tls-certs",
|
|
ReadOnly: true,
|
|
MountPath: "/var/run/kserve/tls",
|
|
},
|
|
},
|
|
LivenessProbe: &corev1.Probe{
|
|
ProbeHandler: corev1.ProbeHandler{
|
|
HTTPGet: &corev1.HTTPGetAction{
|
|
Path: "/health",
|
|
Port: intstr.FromInt32(8000),
|
|
Scheme: corev1.URISchemeHTTP,
|
|
},
|
|
},
|
|
|
|
TimeoutSeconds: 10,
|
|
PeriodSeconds: 10,
|
|
FailureThreshold: 3,
|
|
},
|
|
ReadinessProbe: &corev1.Probe{
|
|
ProbeHandler: corev1.ProbeHandler{
|
|
HTTPGet: &corev1.HTTPGetAction{
|
|
Path: "/health",
|
|
Port: intstr.FromInt32(8000),
|
|
Scheme: corev1.URISchemeHTTP,
|
|
},
|
|
},
|
|
|
|
TimeoutSeconds: 5,
|
|
PeriodSeconds: 10,
|
|
FailureThreshold: 60,
|
|
},
|
|
StartupProbe: &corev1.Probe{
|
|
ProbeHandler: corev1.ProbeHandler{
|
|
HTTPGet: &corev1.HTTPGetAction{
|
|
Path: "/health",
|
|
Port: intstr.FromInt32(8000),
|
|
Scheme: corev1.URISchemeHTTP,
|
|
},
|
|
},
|
|
FailureThreshold: 60,
|
|
PeriodSeconds: 10,
|
|
},
|
|
TerminationMessagePath: "/dev/termination-log",
|
|
TerminationMessagePolicy: "FallbackToLogsOnError",
|
|
ImagePullPolicy: "IfNotPresent",
|
|
SecurityContext: &corev1.SecurityContext{
|
|
Capabilities: &corev1.Capabilities{
|
|
Drop: []corev1.Capability{"ALL"},
|
|
},
|
|
AllowPrivilegeEscalation: ptr.To(false),
|
|
RunAsNonRoot: ptr.To(true),
|
|
ReadOnlyRootFilesystem: ptr.To(true),
|
|
SeccompProfile: &corev1.SeccompProfile{
|
|
Type: corev1.SeccompProfileTypeRuntimeDefault,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
TerminationGracePeriodSeconds: ptr.To(int64(30)),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
remaining := llmisvc.WellKnownDefaultConfigs.Clone()
|
|
|
|
_ = filepath.Walk(presetsDir, func(path string, info os.FileInfo, err error) error {
|
|
if err != nil {
|
|
t.Errorf("Failed to walk %s: %v", path, err)
|
|
return err
|
|
}
|
|
|
|
filename := info.Name()
|
|
if info.IsDir() || !strings.HasSuffix(filename, ".yaml") || !strings.HasPrefix(filename, "config-") {
|
|
return nil
|
|
}
|
|
|
|
t.Run(filename, func(t *testing.T) {
|
|
filePath := filepath.Join(presetsDir, filename)
|
|
data, err := os.ReadFile(filepath.Clean(filePath))
|
|
if err != nil {
|
|
t.Errorf("Failed to read file %s: %v", filePath, err)
|
|
return
|
|
}
|
|
|
|
config := loadConfig(t, data, filePath)
|
|
|
|
name := config.Name
|
|
if !llmisvc.WellKnownDefaultConfigs.Has(name) {
|
|
t.Fatalf("Expected %s to exist in WellKnownDefaultConfigs %#v", name, llmisvc.WellKnownDefaultConfigs)
|
|
}
|
|
// Remove from the tracked set
|
|
remaining = remaining.Delete(name)
|
|
|
|
out, err := llmisvc.ReplaceVariables(llmSvc, config, &kserveSystemConfig)
|
|
if err != nil {
|
|
t.Errorf("ReplaceVariables() failed for %s: %v", filename, err)
|
|
}
|
|
|
|
// Verify the actual Spec rendered if provided for the found file.
|
|
if tc, exist := tt[filename]; exist {
|
|
if !equality.Semantic.DeepDerivative(tc.expected, out) {
|
|
diff := cmp.Diff(tc.expected, out)
|
|
t.Errorf("ReplaceVariables() returned unexpected diff (-want +got):\n%s", diff)
|
|
}
|
|
}
|
|
})
|
|
|
|
return nil
|
|
})
|
|
|
|
if remaining.Len() > 0 {
|
|
t.Errorf("Found %d remaining well-known-configs that are missing as manifest files: %#v", remaining.Len(), remaining)
|
|
}
|
|
}
|
|
|
|
func loadConfig(t *testing.T, data []byte, filePath string) *v1alpha2.LLMInferenceServiceConfig {
|
|
config := &v1alpha2.LLMInferenceServiceConfig{}
|
|
if err := yaml.Unmarshal(data, config); err != nil {
|
|
t.Errorf("Failed to unmarshal YAML from %s: %v", filePath, err)
|
|
return nil
|
|
}
|
|
if err := yaml.Unmarshal(data, config); err != nil {
|
|
t.Errorf("Failed to unmarshal YAML from %s: %v", filePath, err)
|
|
return nil
|
|
}
|
|
|
|
expectedGroupVersion := v1alpha2.LLMInferenceServiceConfigGVK.GroupVersion().String()
|
|
if config.APIVersion != expectedGroupVersion {
|
|
t.Errorf("Expected APIVersion to be '%s', got '%s'", expectedGroupVersion, config.APIVersion)
|
|
}
|
|
|
|
expectedKind := v1alpha2.LLMInferenceServiceConfigGVK.Kind
|
|
if config.Kind != expectedKind {
|
|
t.Errorf("Expected Kind to be '%s', got %s", expectedKind, config.Kind)
|
|
}
|
|
|
|
if config.Name == "" {
|
|
t.Error("Expected ObjectMeta.Name to be set")
|
|
}
|
|
|
|
return config
|
|
}
|