kserve/docs/samples/llmisvc/opt-125m-cpu/llm-inference-service-pd-facebook-opt-125m-cpu.yaml

apiVersion: serving.kserve.io/v1alpha1
kind: LLMInferenceService
metadata:
  name: facebook-opt-125m-pd
spec:
  model:
    uri: hf://facebook/opt-125m
    name: facebook/opt-125m
  router:
    scheduler: { }
    route: { }
    gateway: { }
  template:
    containers:
      - name: main
        image: public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.19.0
        securityContext:
          runAsNonRoot: false
        env:
          - name: VLLM_LOGGING_LEVEL
            value: DEBUG
          - name: VLLM_CPU_KVCACHE_SPACE
            value: "1"
        resources:
          limits:
            cpu: '2'
            memory: 16Gi
          requests:
            cpu: '1'
            memory: 8Gi
  prefill:
    template:
      containers:
        - name: main
          image: public.ecr.aws/q9t5s3a7/vllm-cpu-release-repo:v0.19.0
          env:
            - name: VLLM_LOGGING_LEVEL
              value: DEBUG
          resources:
            limits:
              cpu: '2'
              memory: 16Gi
            requests:
              cpu: '1'
              memory: 8Gi