diff --git a/demo/kserve/Kserve.md b/demo/kserve/Kserve.md index bf001aefc87..63f1dbbd097 100644 --- a/demo/kserve/Kserve.md +++ b/demo/kserve/Kserve.md @@ -87,7 +87,54 @@ sleep 30 oc create -f custom-manifests/opendatahub/kfdef-kserve-op.yaml ~~~ +## Enable metrics for caikit serving + +Note: We are currently enabling metrics by setting a port-level `permissive` mTLS to be able to scrape caikit metrics from the 8086 port of `kserve-container` +### Prerequisites +*cluster-admin is needed to enable user-workload-monitoring* +#### Enable User Workload Monitoring +~~~ +apiVersion: v1 +kind: ConfigMap +metadata: + name: cluster-monitoring-config + namespace: openshift-monitoring +data: + config.yaml: | + enableUserWorkload: true +~~~ +#### Configure User Workload Monitoring +~~~ +apiVersion: v1 +kind: ConfigMap +metadata: + name: user-workload-monitoring-config + namespace: openshift-user-workload-monitoring +data: + config.yaml: | + prometheus: + logLevel: debug + retention: 15d #Change as needed +~~~ +### Adding resources to configure monitoring + +*Resources are created assuming the inferenceservice is deployed in `kserve-demo`. Change files accordingly if the inferenceservice namespace is different* + +``` +oc apply -f ./custom-manifests/metrics/networkpolicy-uwm.yaml +``` +Edit the `peerauthentication-caikit-metrics` file to apply the appropriate `matchLabel` according to your `serving.knative.dev/service` label + +``` +oc apply -f ./custom-manifests/metrics/peerauthentication-caikit-metrics.yaml +``` +``` +oc apply -f ./custom-manifests/metrics/caikit-metrics-service.yaml +oc apply -f ./custom-manifests/metrics/caikit-metrics-servicemonitor.yaml +``` + ## Deploy Minio for example LLM model + ~~~ ACCESS_KEY_ID=THEACCESSKEY SECRET_ACCESS_KEY=$(openssl rand -hex 32) @@ -105,6 +152,7 @@ sed "s//$MINIO_NS/g" ./custom-manifests/minio/serviceaccount-minio.yam If you have installed prerequisites(servicemesh,serverless,kserve and minio), you can start here. ### Setup ISTIO configuration for the test demo namespace + ~~~ export TEST_NS=kserve-demo oc new-project ${TEST_NS} @@ -114,10 +162,13 @@ sed "s//$TEST_NS/g" custom-manifests/service-mesh/peer-authentication-t ~~~ ### Create Caikit ServingRuntime + ~~~ oc apply -f ./custom-manifests/caikit/caikit-servingruntime.yaml ~~~ + ### Deploy example model(flan-t5-samll) + ~~~ oc apply -f ./minio-secret-current.yaml oc create -f ./serviceaccount-minio-current.yaml @@ -131,6 +182,17 @@ export KSVC_HOSTNAME=$(oc get ksvc caikit-example-isvc-predictor -o jsonpath='{. grpcurl -insecure -d '{"text": "At what temperature does liquid Nitrogen boil?"}' -H "mm-model-id: flan-t5-small-caikit" ${KSVC_HOSTNAME}:443 caikit.runtime.Nlp.NlpService/TextGenerationTaskPredict ~~~ +## Verifying Caikit Metrics + +[Prerequisites](#enable-metrics-for-caikit-serving) + +- Navigate to Openshift Console --> Observe --> Targets + - Search by Label `namespace=kserve-demo` + - Verify `caikit-example-isvc-predictor-default-sm` has `status : up` +- Navigate to Openshift Console --> Observe --> Metrics + - Search for `predict_caikit_library_duration_seconds_created` and verify metric values exist + +All caikit produced metrics should successfully show up in Openshift UserWorkload Monitoring now --- **Tip.** diff --git a/demo/kserve/custom-manifests/caikit/caikit-servingruntime.yaml b/demo/kserve/custom-manifests/caikit/caikit-servingruntime.yaml index 65c78634842..39525e056ec 100644 --- a/demo/kserve/custom-manifests/caikit/caikit-servingruntime.yaml +++ b/demo/kserve/custom-manifests/caikit/caikit-servingruntime.yaml @@ -7,8 +7,12 @@ spec: - env: - name: RUNTIME_LOCAL_MODELS_DIR value: /mnt/models + # Use this to enable CPU-only inferencing + # - name: DTYPE_STR + # value: float32 # TODO: This will eventually point to the official image - image: quay.io/opendatahub/caikit-tgis-serving:fast-f23bc24 + # Note: This is currently the latest image, this can change frequently + image: quay.io/opendatahub/caikit-tgis-serving:pr-25 name: kserve-container ports: # Note, KServe only allows a single port, this is the gRPC port. Subject to change in the future diff --git a/demo/kserve/custom-manifests/metrics/caikit-metrics-service.yaml b/demo/kserve/custom-manifests/metrics/caikit-metrics-service.yaml new file mode 100644 index 00000000000..472f8ac44ff --- /dev/null +++ b/demo/kserve/custom-manifests/metrics/caikit-metrics-service.yaml @@ -0,0 +1,16 @@ +kind: Service +apiVersion: v1 +metadata: + name: caikit-example-isvc-predictor-default-sm + namespace: kserve-demo + labels: + name: caikit-example-isvc-predictor-default-sm +spec: + ports: + - name: caikit-metrics + protocol: TCP + port: 8086 + targetPort: 8086 + type: ClusterIP + selector: + serving.knative.dev/service: caikit-example-isvc-predictor-default diff --git a/demo/kserve/custom-manifests/metrics/caikit-metrics-servicemonitor.yaml b/demo/kserve/custom-manifests/metrics/caikit-metrics-servicemonitor.yaml new file mode 100644 index 00000000000..6051347e20e --- /dev/null +++ b/demo/kserve/custom-manifests/metrics/caikit-metrics-servicemonitor.yaml @@ -0,0 +1,13 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: caikit-example-isvc-predictor-default-sm + namespace: kserve-demo +spec: + endpoints: + - port: caikit-metrics + scheme: http + namespaceSelector: {} + selector: + matchLabels: + name: caikit-example-isvc-predictor-default-sm diff --git a/demo/kserve/custom-manifests/metrics/networkpolicy-uwm.yaml b/demo/kserve/custom-manifests/metrics/networkpolicy-uwm.yaml new file mode 100644 index 00000000000..d480868a8a1 --- /dev/null +++ b/demo/kserve/custom-manifests/metrics/networkpolicy-uwm.yaml @@ -0,0 +1,16 @@ +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-from-openshift-monitoring-ns + labels: + app.kubernetes.io/version: release-v1.9 + networking.knative.dev/ingress-provider: istio +spec: + podSelector: {} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: openshift-user-workload-monitoring + policyTypes: + - Ingress diff --git a/demo/kserve/custom-manifests/metrics/peerauthentication-caikit-metrics.yaml b/demo/kserve/custom-manifests/metrics/peerauthentication-caikit-metrics.yaml new file mode 100644 index 00000000000..deb12383879 --- /dev/null +++ b/demo/kserve/custom-manifests/metrics/peerauthentication-caikit-metrics.yaml @@ -0,0 +1,14 @@ +apiVersion: security.istio.io/v1beta1 +kind: PeerAuthentication +metadata: + name: caikit-metrics + namespace: kserve-demo +spec: + mtls: + mode: STRICT + portLevelMtls: + '8086': + mode: PERMISSIVE + selector: + matchLabels: + serving.knative.dev/service: caikit-example-isvc-predictor-default # replace with the appropriate label for your ksvc