Merge branch 'main' into fix-24

demo/kserve/Kserve.md

@@ -87,7 +87,54 @@ sleep 30
 oc create -f custom-manifests/opendatahub/kfdef-kserve-op.yaml
 ~~~
 
+## Enable metrics for caikit serving
+
+Note: We are currently enabling metrics by setting a port-level `permissive` mTLS to be able to scrape caikit metrics from the 8086 port of `kserve-container`
+### Prerequisites
+*cluster-admin is needed to enable user-workload-monitoring*
+#### Enable User Workload Monitoring
+~~~
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cluster-monitoring-config
+  namespace: openshift-monitoring
+data:
+  config.yaml: |
+    enableUserWorkload: true
+~~~
+#### Configure User Workload Monitoring
+~~~
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: user-workload-monitoring-config
+  namespace: openshift-user-workload-monitoring
+data:
+  config.yaml: |
+    prometheus:
+      logLevel: debug 
+      retention: 15d #Change as needed
+~~~
+### Adding resources to configure monitoring
+
+*Resources are created assuming the inferenceservice is deployed in `kserve-demo`. Change files accordingly if the inferenceservice namespace is different*
+
+```
+oc apply -f ./custom-manifests/metrics/networkpolicy-uwm.yaml
+```
+Edit the `peerauthentication-caikit-metrics` file to apply the appropriate `matchLabel` according to your `serving.knative.dev/service` label
+
+```
+oc apply -f ./custom-manifests/metrics/peerauthentication-caikit-metrics.yaml
+```
+```
+oc apply -f ./custom-manifests/metrics/caikit-metrics-service.yaml
+oc apply -f ./custom-manifests/metrics/caikit-metrics-servicemonitor.yaml
+```
+
 ## Deploy Minio for example LLM model
+
 ~~~
 ACCESS_KEY_ID=THEACCESSKEY
 SECRET_ACCESS_KEY=$(openssl rand -hex 32)
@@ -105,6 +152,7 @@ sed "s/<minio_ns>/$MINIO_NS/g" ./custom-manifests/minio/serviceaccount-minio.yam
 If you have installed prerequisites(servicemesh,serverless,kserve and minio), you can start here.
 
 ### Setup ISTIO configuration for the test demo namespace
+
 ~~~
 export TEST_NS=kserve-demo
 oc new-project ${TEST_NS}
@@ -114,10 +162,13 @@ sed "s/<test_ns>/$TEST_NS/g" custom-manifests/service-mesh/peer-authentication-t
 ~~~
 
 ### Create Caikit ServingRuntime
+
 ~~~
 oc apply -f ./custom-manifests/caikit/caikit-servingruntime.yaml
 ~~~
+
 ### Deploy example model(flan-t5-samll)
+
 ~~~
 oc apply -f ./minio-secret-current.yaml 
 oc create -f ./serviceaccount-minio-current.yaml
@@ -131,6 +182,17 @@ export KSVC_HOSTNAME=$(oc get ksvc caikit-example-isvc-predictor -o jsonpath='{.
 grpcurl -insecure -d '{"text": "At what temperature does liquid Nitrogen boil?"}' -H "mm-model-id: flan-t5-small-caikit" ${KSVC_HOSTNAME}:443 caikit.runtime.Nlp.NlpService/TextGenerationTaskPredict
 ~~~
 
+## Verifying Caikit Metrics
+
+[Prerequisites](#enable-metrics-for-caikit-serving)
+
+- Navigate to Openshift Console --> Observe --> Targets
+  - Search by Label `namespace=kserve-demo`
+  - Verify `caikit-example-isvc-predictor-default-sm` has `status : up`
+- Navigate to Openshift Console --> Observe --> Metrics
+  - Search for `predict_caikit_library_duration_seconds_created` and verify metric values exist
+
+All caikit produced metrics should successfully show up in Openshift UserWorkload Monitoring now
 
 ---
 **Tip.**

demo/kserve/custom-manifests/caikit/caikit-servingruntime.yaml

@@ -7,8 +7,12 @@ spec:
   - env:
     - name: RUNTIME_LOCAL_MODELS_DIR
       value: /mnt/models
+    # Use this to enable CPU-only inferencing
+    # - name: DTYPE_STR 
+    #   value: float32
     # TODO: This will eventually point to the official image
-    image: quay.io/opendatahub/caikit-tgis-serving:fast-f23bc24
+    # Note: This is currently the latest image, this can change frequently
+    image: quay.io/opendatahub/caikit-tgis-serving:pr-25
     name: kserve-container
     ports:
     # Note, KServe only allows a single port, this is the gRPC port. Subject to change in the future

demo/kserve/custom-manifests/metrics/caikit-metrics-service.yaml

@@ -0,0 +1,16 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: caikit-example-isvc-predictor-default-sm
+  namespace: kserve-demo
+  labels:
+    name: caikit-example-isvc-predictor-default-sm
+spec:
+  ports:
+    - name: caikit-metrics
+      protocol: TCP
+      port: 8086
+      targetPort: 8086
+  type: ClusterIP
+  selector:
+    serving.knative.dev/service: caikit-example-isvc-predictor-default

demo/kserve/custom-manifests/metrics/caikit-metrics-servicemonitor.yaml

@@ -0,0 +1,13 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: caikit-example-isvc-predictor-default-sm
+  namespace: kserve-demo
+spec:
+  endpoints:
+    - port: caikit-metrics
+      scheme: http
+  namespaceSelector: {}
+  selector:
+    matchLabels:
+      name: caikit-example-isvc-predictor-default-sm

demo/kserve/custom-manifests/metrics/networkpolicy-uwm.yaml

@@ -0,0 +1,16 @@
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+  name: allow-from-openshift-monitoring-ns
+  labels:
+    app.kubernetes.io/version: release-v1.9
+    networking.knative.dev/ingress-provider: istio
+spec:
+  podSelector: {}
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              name: openshift-user-workload-monitoring
+  policyTypes:
+    - Ingress

demo/kserve/custom-manifests/metrics/peerauthentication-caikit-metrics.yaml

@@ -0,0 +1,14 @@
+apiVersion: security.istio.io/v1beta1
+kind: PeerAuthentication
+metadata:
+  name: caikit-metrics
+  namespace: kserve-demo
+spec:
+  mtls:
+    mode: STRICT
+  portLevelMtls:
+    '8086':
+      mode: PERMISSIVE
+  selector:
+    matchLabels:
+      serving.knative.dev/service: caikit-example-isvc-predictor-default # replace with the appropriate label for your ksvc