kserve/config/llmisvcconfig/config-llm-scheduler.yaml

apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
  name: kserve-config-llm-scheduler
spec:
  router:
    scheduler:
      pool:
        spec:
          endpointPickerRef:
            failureMode: FailOpen
            kind: Service
            name: |-
              {{ ChildName .ObjectMeta.Name `-epp-service` }}
            port:
              number: 9002
          selector:
            matchLabels:
              app.kubernetes.io/name: |-
                {{ .ObjectMeta.Name }}
              app.kubernetes.io/part-of: llminferenceservice
              kserve.io/component: workload
          targetPorts:
            - number: 8000
      annotations:
        "app.kubernetes.io/version": "0.9.0"
      template:
        containers:
          - name: main
            lifecycle:
              preStop:
                exec:
                  command:
                  - /bin/sleep
                  - "15"
            ports:
              - containerPort: 9002
                name: grpc
                protocol: TCP
              - containerPort: 9003
                name: grpc-health
                protocol: TCP
              - containerPort: 9090
                name: metrics
                protocol: TCP
              - containerPort: 5557
                name: zmq
                protocol: TCP
            image: ghcr.io/llm-d/llm-d-router-endpoint-picker:v0.9.0-rc.2
            imagePullPolicy: IfNotPresent
            livenessProbe:
              failureThreshold: 3
              grpc:
                port: 9003
                service: liveness
              initialDelaySeconds: 5
              periodSeconds: 10
              successThreshold: 1
              timeoutSeconds: 1
            readinessProbe:
              failureThreshold: 3
              grpc:
                port: 9003
                service: readiness
              initialDelaySeconds: 30
              periodSeconds: 10
              successThreshold: 1
              timeoutSeconds: 1
            command:
              - /app/epp
              - --pool-name
              - "{{ ChildName .ObjectMeta.Name `-inference-pool` }}"
              - --pool-namespace
              - "{{ .ObjectMeta.Namespace }}"
              - --zap-encoder
              - json
              - --grpc-port
              - "9002"
              - --grpc-health-port
              - "9003"
              - '{{ if .GlobalConfig.EnableTLS }}--enable-cert-reload=true{{- end }}'
              - '{{ if .GlobalConfig.EnableTLS }}--secure-serving=true{{- end }}'
              - '{{ if .GlobalConfig.EnableTLS }}--model-server-metrics-scheme=https{{- end }}'
              - '{{ if .GlobalConfig.EnableTLS }}--cert-path=/var/run/kserve/tls{{- end }}'
            env:
              - name: SSL_CERT_DIR
                value: "/var/run/kserve/tls:/var/run/secrets/kubernetes.io/serviceaccount:/etc/pki/tls/certs"
            resources:
              limits:
                cpu: 6
                memory: 16Gi
              requests:
                cpu: 1
                memory: 2Gi
            terminationMessagePath: /dev/termination-log
            terminationMessagePolicy: FallbackToLogsOnError
            securityContext:
              allowPrivilegeEscalation: false
              readOnlyRootFilesystem: true
              runAsNonRoot: true
              capabilities:
                drop:
                  - ALL
              seccompProfile:
                type: RuntimeDefault
            volumeMounts:
              - mountPath: /var/run/kserve/tls
                name: tls-certs
                readOnly: true
              - mountPath: /tmp/tokenizer
                name: tokenizer-uds
          - name: tokenizer
            ports:
              - containerPort: 8082
                name: health
                protocol: TCP
            image: ghcr.io/llm-d/llm-d-uds-tokenizer:vllm-v0.19.1
            imagePullPolicy: IfNotPresent
            workingDir: /mnt/models
            env:
              - name: TOKENIZERS_DIR
                value: /mnt/models
            livenessProbe:
              httpGet:
                path: /healthz
                port: 8082
              periodSeconds: 15
              timeoutSeconds: 5
              failureThreshold: 3
            readinessProbe:
              httpGet:
                path: /healthz
                port: 8082
              periodSeconds: 10
              timeoutSeconds: 5
              failureThreshold: 3
            startupProbe:
              httpGet:
                path: /healthz
                port: 8082
              periodSeconds: 10
              initialDelaySeconds: 5
              timeoutSeconds: 5
              failureThreshold: 60
            resources:
              requests:
                cpu: 256m
                memory: 500Mi
            terminationMessagePath: /dev/termination-log
            terminationMessagePolicy: FallbackToLogsOnError
            securityContext:
              allowPrivilegeEscalation: false
              readOnlyRootFilesystem: true
              runAsNonRoot: true
              capabilities:
                drop:
                  - ALL
              seccompProfile:
                type: RuntimeDefault
            volumeMounts:
              # The tokenizer image imports torch/dill which requires a writable
              # /tmp at import time (tempfile.gettempdir()). Without this mount,
              # readOnlyRootFilesystem causes the container to crash on startup.
              - mountPath: /tmp
                name: tokenizer-tmp
              # vllm writes model info cache to /.cache/vllm/modelinfos.
              - mountPath: /.cache
                name: tokenizer-cache
              - mountPath: /tmp/tokenizer
                name: tokenizer-uds
        volumes:
          - name: tls-certs
            secret:
              secretName: "{{ ChildName .ObjectMeta.Name `-kserve-self-signed-certs` }}"
          - name: tokenizer-uds
            emptyDir: {}
          # Writable /tmp for the tokenizer container (see volumeMounts comment above).
          - name: tokenizer-tmp
            emptyDir: {}
          # Writable /.cache for vllm model info cache.
          - name: tokenizer-cache
            emptyDir: {}
        dnsPolicy: ClusterFirst
        restartPolicy: Always
        terminationGracePeriodSeconds: 60 # must be >= workload terminationGracePeriodSeconds