mirror of
https://github.com/hanzo-ml/kserve.git
synced 2026-10-30 20:53:49 +00:00
SivananthamandGitHub
2a534af9fd
* Add client sdk for localmodelcache, localmodelnodegroup Signed-off-by: Sivanantham Chinnaiyan <[email protected]> * Add e2e test for modelcache Signed-off-by: Sivanantham Chinnaiyan <[email protected]> * Use docker driver and minikube tunnel Signed-off-by: Sivanantham Chinnaiyan <[email protected]> * Merge "Allow multiple node groups in the model cache CR (#4134)" Signed-off-by: Sivanantham Chinnaiyan <[email protected]> * Try mounting image dir Signed-off-by: Sivanantham Chinnaiyan <[email protected]> * Add local model agent to image scan Signed-off-by: Sivanantham Chinnaiyan <[email protected]> * Debug Signed-off-by: Sivanantham Chinnaiyan <[email protected]> * Create model root directory beforehand Signed-off-by: Sivanantham Chinnaiyan <[email protected]> * Restart kserve controller after patch Signed-off-by: Sivanantham Chinnaiyan <[email protected]> * Enablepvc direct mount in e2e test Signed-off-by: Sivanantham Chinnaiyan <[email protected]> * Reduce pv storage to 1GB Signed-off-by: Sivanantham Chinnaiyan <[email protected]> * Update modelcache test Signed-off-by: Sivanantham Chinnaiyan <[email protected]> * Update status-check to include modelcache logs Signed-off-by: Sivanantham Chinnaiyan <[email protected]> --------- Signed-off-by: Sivanantham Chinnaiyan <[email protected]>
357 lines
12 KiB
Python
357 lines
12 KiB
Python
# Copyright 2022 The KServe Authors.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
import json
|
|
import os
|
|
|
|
import pytest
|
|
from kubernetes import client
|
|
from kubernetes.client import V1ContainerPort, V1EnvVar, V1ResourceRequirements
|
|
|
|
from kserve import (
|
|
KServeClient,
|
|
V1beta1InferenceService,
|
|
V1beta1InferenceServiceSpec,
|
|
V1beta1ModelFormat,
|
|
V1beta1ModelSpec,
|
|
V1beta1PredictorSpec,
|
|
V1beta1XGBoostSpec,
|
|
constants,
|
|
)
|
|
|
|
from ..common.utils import KSERVE_TEST_NAMESPACE, predict_isvc, predict_grpc
|
|
|
|
|
|
@pytest.mark.predictor
|
|
@pytest.mark.path_based_routing
|
|
@pytest.mark.asyncio(scope="session")
|
|
async def test_xgboost_kserve(rest_v1_client):
|
|
service_name = "isvc-xgboost"
|
|
predictor = V1beta1PredictorSpec(
|
|
min_replicas=1,
|
|
xgboost=V1beta1XGBoostSpec(
|
|
storage_uri="gs://kfserving-examples/models/xgboost/1.5/model",
|
|
resources=V1ResourceRequirements(
|
|
requests={"cpu": "50m", "memory": "128Mi"},
|
|
limits={"cpu": "100m", "memory": "256Mi"},
|
|
),
|
|
),
|
|
)
|
|
|
|
isvc = V1beta1InferenceService(
|
|
api_version=constants.KSERVE_V1BETA1,
|
|
kind=constants.KSERVE_KIND_INFERENCESERVICE,
|
|
metadata=client.V1ObjectMeta(
|
|
name=service_name, namespace=KSERVE_TEST_NAMESPACE
|
|
),
|
|
spec=V1beta1InferenceServiceSpec(predictor=predictor),
|
|
)
|
|
|
|
kserve_client = KServeClient(
|
|
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
|
|
)
|
|
kserve_client.create(isvc)
|
|
kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE)
|
|
res = await predict_isvc(rest_v1_client, service_name, "./data/iris_input.json")
|
|
assert res["predictions"] == [1, 1]
|
|
kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE)
|
|
|
|
|
|
@pytest.mark.predictor
|
|
@pytest.mark.path_based_routing
|
|
@pytest.mark.asyncio(scope="session")
|
|
async def test_xgboost_v2_mlserver(rest_v2_client):
|
|
service_name = "isvc-xgboost-v2-mlserver"
|
|
protocol_version = "v2"
|
|
|
|
predictor = V1beta1PredictorSpec(
|
|
min_replicas=1,
|
|
xgboost=V1beta1XGBoostSpec(
|
|
storage_uri="gs://kfserving-examples/models/xgboost/iris",
|
|
env=[V1EnvVar(name="MLSERVER_MODEL_PARALLEL_WORKERS", value="0")],
|
|
protocol_version=protocol_version,
|
|
resources=V1ResourceRequirements(
|
|
requests={"cpu": "50m", "memory": "128Mi"},
|
|
limits={"cpu": "100m", "memory": "1024Mi"},
|
|
),
|
|
readiness_probe=client.V1Probe(
|
|
http_get=client.V1HTTPGetAction(
|
|
path=f"/v2/models/{service_name}/ready", port=8080
|
|
),
|
|
initial_delay_seconds=30,
|
|
),
|
|
),
|
|
)
|
|
|
|
isvc = V1beta1InferenceService(
|
|
api_version=constants.KSERVE_V1BETA1,
|
|
kind=constants.KSERVE_KIND_INFERENCESERVICE,
|
|
metadata=client.V1ObjectMeta(
|
|
name=service_name, namespace=KSERVE_TEST_NAMESPACE
|
|
),
|
|
spec=V1beta1InferenceServiceSpec(predictor=predictor),
|
|
)
|
|
|
|
kserve_client = KServeClient(
|
|
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
|
|
)
|
|
kserve_client.create(isvc)
|
|
kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE)
|
|
|
|
res = await predict_isvc(
|
|
rest_v2_client,
|
|
service_name,
|
|
"./data/iris_input_v2.json",
|
|
)
|
|
assert res.outputs[0].data == [1.0, 1.0]
|
|
|
|
kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE)
|
|
|
|
|
|
@pytest.mark.predictor
|
|
@pytest.mark.path_based_routing
|
|
@pytest.mark.asyncio(scope="session")
|
|
async def test_xgboost_single_model_file(rest_v2_client):
|
|
service_name = "xgboost-v2-mlserver"
|
|
protocol_version = "v2"
|
|
|
|
predictor = V1beta1PredictorSpec(
|
|
min_replicas=1,
|
|
xgboost=V1beta1XGBoostSpec(
|
|
storage_uri="gs://kfserving-examples/models/xgboost/iris/model.bst",
|
|
env=[V1EnvVar(name="MLSERVER_MODEL_PARALLEL_WORKERS", value="0")],
|
|
protocol_version=protocol_version,
|
|
resources=V1ResourceRequirements(
|
|
requests={"cpu": "50m", "memory": "128Mi"},
|
|
limits={"cpu": "100m", "memory": "1024Mi"},
|
|
),
|
|
readiness_probe=client.V1Probe(
|
|
http_get=client.V1HTTPGetAction(
|
|
path=f"/v2/models/{service_name}/ready", port=8080
|
|
),
|
|
initial_delay_seconds=30,
|
|
),
|
|
),
|
|
)
|
|
|
|
isvc = V1beta1InferenceService(
|
|
api_version=constants.KSERVE_V1BETA1,
|
|
kind=constants.KSERVE_KIND_INFERENCESERVICE,
|
|
metadata=client.V1ObjectMeta(
|
|
name=service_name, namespace=KSERVE_TEST_NAMESPACE
|
|
),
|
|
spec=V1beta1InferenceServiceSpec(predictor=predictor),
|
|
)
|
|
|
|
kserve_client = KServeClient(
|
|
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
|
|
)
|
|
kserve_client.create(isvc)
|
|
kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE)
|
|
|
|
res = await predict_isvc(
|
|
rest_v2_client,
|
|
service_name,
|
|
"./data/iris_input_v2.json",
|
|
)
|
|
assert res.outputs[0].data == [1.0, 1.0]
|
|
|
|
kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE)
|
|
|
|
|
|
@pytest.mark.predictor
|
|
@pytest.mark.path_based_routing
|
|
@pytest.mark.asyncio(scope="session")
|
|
async def test_xgboost_runtime_kserve(rest_v1_client):
|
|
service_name = "isvc-xgboost-runtime"
|
|
predictor = V1beta1PredictorSpec(
|
|
min_replicas=1,
|
|
model=V1beta1ModelSpec(
|
|
model_format=V1beta1ModelFormat(
|
|
name="xgboost",
|
|
),
|
|
storage_uri="gs://kfserving-examples/models/xgboost/1.5/model",
|
|
resources=V1ResourceRequirements(
|
|
requests={"cpu": "50m", "memory": "128Mi"},
|
|
limits={"cpu": "100m", "memory": "256Mi"},
|
|
),
|
|
),
|
|
)
|
|
|
|
isvc = V1beta1InferenceService(
|
|
api_version=constants.KSERVE_V1BETA1,
|
|
kind=constants.KSERVE_KIND_INFERENCESERVICE,
|
|
metadata=client.V1ObjectMeta(
|
|
name=service_name, namespace=KSERVE_TEST_NAMESPACE
|
|
),
|
|
spec=V1beta1InferenceServiceSpec(predictor=predictor),
|
|
)
|
|
|
|
kserve_client = KServeClient(
|
|
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
|
|
)
|
|
kserve_client.create(isvc)
|
|
kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE)
|
|
res = await predict_isvc(rest_v1_client, service_name, "./data/iris_input.json")
|
|
assert res["predictions"] == [1, 1]
|
|
kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE)
|
|
|
|
|
|
@pytest.mark.predictor
|
|
@pytest.mark.path_based_routing
|
|
@pytest.mark.asyncio(scope="session")
|
|
async def test_xgboost_v2_runtime_mlserver(rest_v2_client):
|
|
service_name = "isvc-xgboost-v2-runtime"
|
|
protocol_version = "v2"
|
|
|
|
predictor = V1beta1PredictorSpec(
|
|
min_replicas=1,
|
|
model=V1beta1ModelSpec(
|
|
model_format=V1beta1ModelFormat(
|
|
name="xgboost",
|
|
),
|
|
runtime="kserve-mlserver",
|
|
storage_uri="gs://kfserving-examples/models/xgboost/iris",
|
|
protocol_version=protocol_version,
|
|
resources=V1ResourceRequirements(
|
|
requests={"cpu": "50m", "memory": "128Mi"},
|
|
limits={"cpu": "100m", "memory": "1024Mi"},
|
|
),
|
|
readiness_probe=client.V1Probe(
|
|
http_get=client.V1HTTPGetAction(
|
|
path=f"/v2/models/{service_name}/ready", port=8080
|
|
),
|
|
initial_delay_seconds=30,
|
|
),
|
|
),
|
|
)
|
|
|
|
isvc = V1beta1InferenceService(
|
|
api_version=constants.KSERVE_V1BETA1,
|
|
kind=constants.KSERVE_KIND_INFERENCESERVICE,
|
|
metadata=client.V1ObjectMeta(
|
|
name=service_name, namespace=KSERVE_TEST_NAMESPACE
|
|
),
|
|
spec=V1beta1InferenceServiceSpec(predictor=predictor),
|
|
)
|
|
|
|
kserve_client = KServeClient(
|
|
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
|
|
)
|
|
kserve_client.create(isvc)
|
|
kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE)
|
|
|
|
res = await predict_isvc(
|
|
rest_v2_client,
|
|
service_name,
|
|
"./data/iris_input_v2.json",
|
|
)
|
|
assert res.outputs[0].data == [1.0, 1.0]
|
|
|
|
kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE)
|
|
|
|
|
|
@pytest.mark.predictor
|
|
@pytest.mark.path_based_routing
|
|
@pytest.mark.asyncio(scope="session")
|
|
async def test_xgboost_v2(rest_v2_client):
|
|
service_name = "isvc-xgboost-v2"
|
|
predictor = V1beta1PredictorSpec(
|
|
min_replicas=1,
|
|
model=V1beta1ModelSpec(
|
|
model_format=V1beta1ModelFormat(
|
|
name="xgboost",
|
|
),
|
|
runtime="kserve-xgbserver",
|
|
storage_uri="gs://kfserving-examples/models/xgboost/iris",
|
|
resources=V1ResourceRequirements(
|
|
requests={"cpu": "50m", "memory": "128Mi"},
|
|
limits={"cpu": "100m", "memory": "1024Mi"},
|
|
),
|
|
),
|
|
)
|
|
|
|
isvc = V1beta1InferenceService(
|
|
api_version=constants.KSERVE_V1BETA1,
|
|
kind=constants.KSERVE_KIND_INFERENCESERVICE,
|
|
metadata=client.V1ObjectMeta(
|
|
name=service_name, namespace=KSERVE_TEST_NAMESPACE
|
|
),
|
|
spec=V1beta1InferenceServiceSpec(predictor=predictor),
|
|
)
|
|
|
|
kserve_client = KServeClient(
|
|
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
|
|
)
|
|
kserve_client.create(isvc)
|
|
kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE)
|
|
|
|
res = await predict_isvc(
|
|
rest_v2_client,
|
|
service_name,
|
|
"./data/iris_input_v2.json",
|
|
)
|
|
assert res.outputs[0].data == [1.0, 1.0]
|
|
|
|
kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE)
|
|
|
|
|
|
@pytest.mark.grpc
|
|
@pytest.mark.predictor
|
|
@pytest.mark.asyncio(scope="session")
|
|
async def test_xgboost_v2_grpc():
|
|
service_name = "isvc-xgboost-v2-grpc"
|
|
model_name = "xgboost"
|
|
predictor = V1beta1PredictorSpec(
|
|
min_replicas=1,
|
|
model=V1beta1ModelSpec(
|
|
model_format=V1beta1ModelFormat(
|
|
name="xgboost",
|
|
),
|
|
runtime="kserve-xgbserver",
|
|
storage_uri="gs://kfserving-examples/models/xgboost/iris",
|
|
resources=V1ResourceRequirements(
|
|
requests={"cpu": "50m", "memory": "128Mi"},
|
|
limits={"cpu": "100m", "memory": "1024Mi"},
|
|
),
|
|
ports=[V1ContainerPort(container_port=8081, name="h2c", protocol="TCP")],
|
|
args=["--model_name", model_name],
|
|
),
|
|
)
|
|
|
|
isvc = V1beta1InferenceService(
|
|
api_version=constants.KSERVE_V1BETA1,
|
|
kind=constants.KSERVE_KIND_INFERENCESERVICE,
|
|
metadata=client.V1ObjectMeta(
|
|
name=service_name, namespace=KSERVE_TEST_NAMESPACE
|
|
),
|
|
spec=V1beta1InferenceServiceSpec(predictor=predictor),
|
|
)
|
|
|
|
kserve_client = KServeClient(
|
|
config_file=os.environ.get("KUBECONFIG", "~/.kube/config")
|
|
)
|
|
kserve_client.create(isvc)
|
|
kserve_client.wait_isvc_ready(service_name, namespace=KSERVE_TEST_NAMESPACE)
|
|
|
|
json_file = open("./data/iris_input_v2_grpc.json")
|
|
payload = json.load(json_file)["inputs"]
|
|
response = await predict_grpc(
|
|
service_name=service_name, payload=payload, model_name=model_name
|
|
)
|
|
prediction = response.outputs[0].data
|
|
assert prediction == [1.0, 1.0]
|
|
|
|
kserve_client.delete(service_name, KSERVE_TEST_NAMESPACE)
|