diff --git a/demo/kserve/.gitignore b/demo/kserve/.gitignore index 5e49500fc31..661af40fa3b 100644 --- a/demo/kserve/.gitignore +++ b/demo/kserve/.gitignore @@ -1,7 +1,6 @@ kserve/ knative-kserve/ odh-manifests/ -minio-current.yaml -minio-secret-current.yaml +*-current.yaml .cache/ kustomize/ diff --git a/demo/kserve/Kserve.md b/demo/kserve/Kserve.md index 2b8210c263f..63f1dbbd097 100644 --- a/demo/kserve/Kserve.md +++ b/demo/kserve/Kserve.md @@ -18,8 +18,7 @@ - https://github.com/ReToCode/knative-kserve#installation-with-istio--mesh - https://knative.dev/docs/install/operator/knative-with-operators/#create-the-knative-serving-custom-resource -## Steps -### Prerequisite installation +## Prerequisite installation ~~~ git clone https://github.com/opendatahub-io/caikit-tgis-serving cd caikit-tgis-serving/demo/kserve @@ -43,10 +42,10 @@ oc wait --for=condition=ready pod -l app=jaeger -n istio-system --timeout=300s # kserve/knative oc create ns kserve -oc create ns kserve-demo oc create ns knative-serving -oc apply -f custom-manifests/service-mesh/smmr.yaml -oc apply -f custom-manifests/service-mesh/peer-authentication.yaml # we need this because of https://access.redhat.com/documentation/en-us/openshift_container_platform/4.12/html/serverless/serving#serverless-domain-mapping-custom-tls-cert_domain-mapping-custom-tls-cert +sed "s//$TEST_NS/g" custom-manifests/service-mesh/smmr.yaml | tee ./smmr-current.yaml | oc -n istio-system apply -f - +sed "s//$TEST_NS/g" custom-manifests/service-mesh/peer-authentication.yaml | tee ./peer-authentication-current.yaml | oc apply -f - +# we need this because of https://access.redhat.com/documentation/en-us/openshift_container_platform/4.12/html/serverless/serving#serverless-domain-mapping-custom-tls-cert_domain-mapping-custom-tls-cert oc apply -f custom-manifests/serverless/operators.yaml sleep 30 @@ -134,29 +133,50 @@ oc apply -f ./custom-manifests/metrics/caikit-metrics-service.yaml oc apply -f ./custom-manifests/metrics/caikit-metrics-servicemonitor.yaml ``` +## Deploy Minio for example LLM model -## Deploy flan-t5-small model with Caikit+TGIS Serving runtime ~~~ -# Minio Deploy ACCESS_KEY_ID=THEACCESSKEY SECRET_ACCESS_KEY=$(openssl rand -hex 32) +MINIO_NS=minio + +oc new-project ${MINIO_NS} +sed "s//$ACCESS_KEY_ID/g" ./custom-manifests/minio/minio.yaml | sed "s++$SECRET_ACCESS_KEY+g" | tee ./minio-current.yaml | oc -n ${MINIO_NS} apply -f - +sed "s//$ACCESS_KEY_ID/g" ./custom-manifests/minio/minio-secret.yaml | sed "s++$SECRET_ACCESS_KEY+g" |sed "s//$MINIO_NS/g" | tee ./minio-secret-current.yaml | oc -n ${MINIO_NS} apply -f - + +sed "s//$MINIO_NS/g" ./custom-manifests/minio/serviceaccount-minio.yaml | tee ./serviceaccount-minio-current.yaml +~~~ -oc new-project minio -sed "s//$ACCESS_KEY_ID/g" ./custom-manifests/minio/minio.yaml | sed "s++$SECRET_ACCESS_KEY+g" | tee ./minio-current.yaml | oc -n minio apply -f - -sed "s//$ACCESS_KEY_ID/g" ./custom-manifests/minio/minio-secret.yaml | sed "s++$SECRET_ACCESS_KEY+g" | tee ./minio-secret-current.yaml | oc -n minio apply -f - +## Deploy flan-t5-small model with Caikit+TGIS Serving runtime + +If you have installed prerequisites(servicemesh,serverless,kserve and minio), you can start here. -# Create Caikit Serving runtime -oc project kserve-demo +### Setup ISTIO configuration for the test demo namespace + +~~~ +export TEST_NS=kserve-demo +oc new-project ${TEST_NS} +sed "s//$TEST_NS/g" custom-manifests/service-mesh/smmr-test-ns.yaml | tee ./smmr-current.yaml | oc -n istio-system apply -f - +sed "s//$TEST_NS/g" custom-manifests/service-mesh/peer-authentication-tests-ns.yaml | tee ./peer-authentication-test-ns-current.yaml | oc apply -f - +# we need this because of https://access.redhat.com/documentation/en-us/openshift_container_platform/4.12/html/serverless/serving#serverless-domain-mapping-custom-tls-cert_domain-mapping-custom-tls-cert +~~~ + +### Create Caikit ServingRuntime + +~~~ oc apply -f ./custom-manifests/caikit/caikit-servingruntime.yaml +~~~ + +### Deploy example model(flan-t5-samll) -# Deploy model +~~~ oc apply -f ./minio-secret-current.yaml -oc create -f ./custom-manifests/minio/serviceaccount-minio.yaml +oc create -f ./serviceaccount-minio-current.yaml -oc apply -f ./custom-manifests/caikit/caikit-isvc.yaml -n kserve-demo +oc apply -f ./custom-manifests/caikit/caikit-isvc.yaml -n ${TEST_NS} ~~~ -## gRPC Test +### gRPC Test ~~~ export KSVC_HOSTNAME=$(oc get ksvc caikit-example-isvc-predictor -o jsonpath='{.status.url}' | cut -d'/' -f3) grpcurl -insecure -d '{"text": "At what temperature does liquid Nitrogen boil?"}' -H "mm-model-id: flan-t5-small-caikit" ${KSVC_HOSTNAME}:443 caikit.runtime.Nlp.NlpService/TextGenerationTaskPredict diff --git a/demo/kserve/custom-manifests/minio/minio-secret.yaml b/demo/kserve/custom-manifests/minio/minio-secret.yaml index bea53792717..887d393c760 100644 --- a/demo/kserve/custom-manifests/minio/minio-secret.yaml +++ b/demo/kserve/custom-manifests/minio/minio-secret.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: annotations: - serving.kserve.io/s3-endpoint: minio.minio.svc:9000 # replace with your s3 endpoint e.g minio-service.kubeflow:9000 + serving.kserve.io/s3-endpoint: minio..svc:9000 # replace with your s3 endpoint e.g minio-service.kubeflow:9000 serving.kserve.io/s3-usehttps: "0" # by default 1, if testing with minio you can set to 0 serving.kserve.io/s3-region: "us-east-2" serving.kserve.io/s3-useanoncredential: "false" # omitting this is the same as false, if true will ignore provided credential and use anonymous credentials diff --git a/demo/kserve/custom-manifests/minio/serviceaccount-minio.yaml b/demo/kserve/custom-manifests/minio/serviceaccount-minio.yaml index ece45947af6..47be8158f09 100644 --- a/demo/kserve/custom-manifests/minio/serviceaccount-minio.yaml +++ b/demo/kserve/custom-manifests/minio/serviceaccount-minio.yaml @@ -3,7 +3,7 @@ kind: ServiceAccount metadata: name: sa annotations: - serving.kserve.io/s3-endpoint: minio.minio.svc:9000 # replace with your s3 endpoint e.g minio-service.kubeflow:9000 + serving.kserve.io/s3-endpoint: minio..svc:9000 # replace with your s3 endpoint e.g minio-service.kubeflow:9000 serving.kserve.io/s3-usehttps: "0" # by default 1, if testing with minio you can set to 0 serving.kserve.io/s3-region: "us-east-2" serving.kserve.io/s3-useanoncredential: "false" # omitting this is the same as false, if true will ignore provided credential and use anonymous credentials diff --git a/demo/kserve/custom-manifests/service-mesh/peer-authentication-test-ns.yaml b/demo/kserve/custom-manifests/service-mesh/peer-authentication-test-ns.yaml new file mode 100644 index 00000000000..9058c8ac7fb --- /dev/null +++ b/demo/kserve/custom-manifests/service-mesh/peer-authentication-test-ns.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: security.istio.io/v1beta1 +kind: PeerAuthentication +metadata: + name: default + namespace: +spec: + mtls: + mode: STRICT diff --git a/demo/kserve/custom-manifests/service-mesh/peer-authentication.yaml b/demo/kserve/custom-manifests/service-mesh/peer-authentication.yaml index 6b2694c3c13..b57c9273cdd 100644 --- a/demo/kserve/custom-manifests/service-mesh/peer-authentication.yaml +++ b/demo/kserve/custom-manifests/service-mesh/peer-authentication.yaml @@ -24,13 +24,3 @@ metadata: spec: mtls: mode: STRICT ---- -apiVersion: security.istio.io/v1beta1 -kind: PeerAuthentication -metadata: - name: default - namespace: kserve-demo -spec: - mtls: - mode: STRICT ---- diff --git a/demo/kserve/custom-manifests/service-mesh/smmr-test-ns.yaml b/demo/kserve/custom-manifests/service-mesh/smmr-test-ns.yaml new file mode 100644 index 00000000000..f8cc3359c04 --- /dev/null +++ b/demo/kserve/custom-manifests/service-mesh/smmr-test-ns.yaml @@ -0,0 +1,8 @@ +apiVersion: maistra.io/v1 +kind: ServiceMeshMemberRoll +metadata: + name: default + namespace: istio-system +spec: + members: + - diff --git a/demo/kserve/custom-manifests/service-mesh/smmr.yaml b/demo/kserve/custom-manifests/service-mesh/smmr.yaml index 2e7724fe465..87cc15414ec 100644 --- a/demo/kserve/custom-manifests/service-mesh/smmr.yaml +++ b/demo/kserve/custom-manifests/service-mesh/smmr.yaml @@ -7,5 +7,4 @@ spec: members: - knative-serving - kserve - - kserve-demo - - auth-provider + -