kserve/config/llmisvcconfig/config-llm-router-route.yaml

# HTTPRoute template for LLMInferenceService routing.
#
# Each endpoint (completions, chat/completions, responses, messages) is split into two
# rules because Gateway API requires exactly one PathPrefix match per rule
# when using URLRewrite with replacePrefixMatch:
#
#   1. PathPrefix rule  — matches /{namespace}/{name}/v1/... and rewrites to
#      /v1/..., routing through the InferencePool. Carries the URLRewrite filter.
#
#   2. Header-based rule — matches the exact path /v1/... with the model-routing
#      header (ModelBasedRoutingHeaderName), enabling shared-gateway deployments
#      where a single gateway dispatches to multiple models by header value.
#      No URLRewrite is needed since the path is already the target path.
#
# The catch-all Service rules follow the same split: a PathPrefix rule with
# URLRewrite for path-based access, and a header-only rule for model-based routing.
apiVersion: serving.kserve.io/v1alpha2
kind: LLMInferenceServiceConfig
metadata:
  name: kserve-config-llm-router-route
spec:
  router:
    route:
      http:
        spec:
          parentRefs:
            - group: gateway.networking.k8s.io
              kind: Gateway
              name: |-
                {{ .GlobalConfig.IngressGatewayName }}
              namespace: |-
                {{ .GlobalConfig.IngressGatewayNamespace }}
          rules:
            - name: v1-completions-path
              backendRefs:
                - group: inference.networking.k8s.io
                  kind: InferencePool
                  name: |-
                    {{ ChildName .ObjectMeta.Name `-inference-pool` }}
                  port: 8000
                  weight: 1
              matches:
                - path:
                    type: PathPrefix
                    value: |-
                      /{{ .ObjectMeta.Namespace }}/{{ .ObjectMeta.Name }}/v1/completions
              filters:
                - type: URLRewrite
                  urlRewrite:
                    path:
                      type: ReplacePrefixMatch
                      replacePrefixMatch: /v1/completions
              timeouts:
                backendRequest: 0s
                request: 0s
            - name: v1-completions-model-routing
              backendRefs:
                - group: inference.networking.k8s.io
                  kind: InferencePool
                  name: |-
                    {{ ChildName .ObjectMeta.Name `-inference-pool` }}
                  port: 8000
                  weight: 1
              matches:
                - path:
                    type: Exact
                    value: /v1/completions
                  headers:
                    - type: Exact
                      name: "{{ .GlobalConfig.ModelBasedRoutingHeaderName }}"
                      value: publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name }}
                - path:
                    type: Exact
                    value: /v1/completions/
                  headers:
                    - type: Exact
                      name: "{{ .GlobalConfig.ModelBasedRoutingHeaderName }}"
                      value: publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name }}
              timeouts:
                backendRequest: 0s
                request: 0s
            - name: v1-chat-completions-path
              backendRefs:
                - group: inference.networking.k8s.io
                  kind: InferencePool
                  name: |-
                    {{ ChildName .ObjectMeta.Name `-inference-pool` }}
                  port: 8000
                  weight: 1
              matches:
                - path:
                    type: PathPrefix
                    value: |-
                      /{{ .ObjectMeta.Namespace }}/{{ .ObjectMeta.Name }}/v1/chat/completions
              filters:
                - type: URLRewrite
                  urlRewrite:
                    path:
                      type: ReplacePrefixMatch
                      replacePrefixMatch: /v1/chat/completions
              timeouts:
                backendRequest: 0s
                request: 0s
            - name: v1-chat-completions-model-routing
              backendRefs:
                - group: inference.networking.k8s.io
                  kind: InferencePool
                  name: |-
                    {{ ChildName .ObjectMeta.Name `-inference-pool` }}
                  port: 8000
                  weight: 1
              matches:
                - path:
                    type: Exact
                    value: /v1/chat/completions
                  headers:
                    - type: Exact
                      name: "{{ .GlobalConfig.ModelBasedRoutingHeaderName }}"
                      value: publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name }}
                - path:
                    type: Exact
                    value: /v1/chat/completions/
                  headers:
                    - type: Exact
                      name: "{{ .GlobalConfig.ModelBasedRoutingHeaderName }}"
                      value: publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name }}
              timeouts:
                backendRequest: 0s
                request: 0s
            - name: v1-responses-path
              backendRefs:
                - group: inference.networking.k8s.io
                  kind: InferencePool
                  name: |-
                    {{ ChildName .ObjectMeta.Name `-inference-pool` }}
                  port: 8000
                  weight: 1
              matches:
                - path:
                    type: PathPrefix
                    value: |-
                      /{{ .ObjectMeta.Namespace }}/{{ .ObjectMeta.Name }}/v1/responses
              filters:
                - type: URLRewrite
                  urlRewrite:
                    path:
                      type: ReplacePrefixMatch
                      replacePrefixMatch: /v1/responses
              timeouts:
                backendRequest: 0s
                request: 0s
            - name: v1-responses-model-routing
              backendRefs:
                - group: inference.networking.k8s.io
                  kind: InferencePool
                  name: |-
                    {{ ChildName .ObjectMeta.Name `-inference-pool` }}
                  port: 8000
                  weight: 1
              matches:
                - path:
                    type: Exact
                    value: /v1/responses
                  headers:
                    - type: Exact
                      name: "{{ .GlobalConfig.ModelBasedRoutingHeaderName }}"
                      value: publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name }}
                - path:
                    type: Exact
                    value: /v1/responses/
                  headers:
                    - type: Exact
                      name: "{{ .GlobalConfig.ModelBasedRoutingHeaderName }}"
                      value: publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name }}
              timeouts:
                backendRequest: 0s
                request: 0s
            - name: v1-messages-path
              backendRefs:
                - group: inference.networking.k8s.io
                  kind: InferencePool
                  name: |-
                    {{ ChildName .ObjectMeta.Name `-inference-pool` }}
                  port: 8000
                  weight: 1
              matches:
                - path:
                    type: PathPrefix
                    value: |-
                      /{{ .ObjectMeta.Namespace }}/{{ .ObjectMeta.Name }}/v1/messages
              filters:
                - type: URLRewrite
                  urlRewrite:
                    path:
                      type: ReplacePrefixMatch
                      replacePrefixMatch: /v1/messages
              timeouts:
                backendRequest: 0s
                request: 0s
            - name: v1-messages-model-routing
              backendRefs:
                - group: inference.networking.k8s.io
                  kind: InferencePool
                  name: |-
                    {{ ChildName .ObjectMeta.Name `-inference-pool` }}
                  port: 8000
                  weight: 1
              matches:
                - path:
                    type: Exact
                    value: /v1/messages
                  headers:
                    - type: Exact
                      name: "{{ .GlobalConfig.ModelBasedRoutingHeaderName }}"
                      value: publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name }}
                - path:
                    type: Exact
                    value: /v1/messages/
                  headers:
                    - type: Exact
                      name: "{{ .GlobalConfig.ModelBasedRoutingHeaderName }}"
                      value: publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name }}
              timeouts:
                backendRequest: 0s
                request: 0s
            # Catch-all rules route to the workload Service directly (bypassing
            # InferencePool) for endpoints not handled by the scheduler, such as
            # /v1/models, /health, or custom paths exposed by the runtime.
            - name: v1-catch-all-path
              backendRefs:
                - kind: Service
                  name: |-
                    {{ ChildName .ObjectMeta.Name `-kserve-workload-svc` }}
                  port: 8000
                  weight: 1
              matches:
                - path:
                    type: PathPrefix
                    value: |-
                      /{{ .ObjectMeta.Namespace }}/{{ .ObjectMeta.Name }}
              filters:
                - type: URLRewrite
                  urlRewrite:
                    path:
                      type: ReplacePrefixMatch
                      replacePrefixMatch: /
              timeouts:
                backendRequest: 0s
                request: 0s
            - name: v1-catch-all-model-routing
              backendRefs:
                - kind: Service
                  name: |-
                    {{ ChildName .ObjectMeta.Name `-kserve-workload-svc` }}
                  port: 8000
                  weight: 1
              matches:
                - headers:
                    - type: Exact
                      name: "{{ .GlobalConfig.ModelBasedRoutingHeaderName }}"
                      value: publishers/{{ .ObjectMeta.Namespace }}/models/{{ .Spec.Model.Name }}
              timeouts:
                backendRequest: 0s
                request: 0s