kubeai: enableMonitoring toggle for Prometheus monitoring of vLLM engine

QC-Labs · Feb 15, 2025 · 55b72bb · 55b72bb
1 parent fc7ee5c
commit 55b72bb
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 0 deletions.
diff --git a/Pulumi.yaml b/Pulumi.yaml
@@ -105,6 +105,7 @@ config:
     kubeai:enabled: false
     kubeai:hostname: kubeai
     kubeai:preferredNodeLabel: orangelab/kubeai
+    kubeai:enableMonitoring: false
     # Set token to access gated repos
     # kubeai:huggingfaceToken: ''
     # Comma-separated list of models from https://github.com/substratusai/kubeai/blob/main/charts/models/values.yaml

diff --git a/components/ai/AI.md b/components/ai/AI.md
@@ -243,6 +243,16 @@ pulumi config set --secret kubeai:huggingfaceToken <hf_token>
 pulumi up
 ```
 
+Prometheus monitoring for vLLM can be enabled with:
+
+```sh
+pulumi set kubeai:enableMonitoring true
+```
+
+Note Prometheus has to be installed first before changing that switch.
+
+Sample Grafana dashboard to import - https://github.com/substratusai/kubeai/blob/main/examples/observability/vllm-grafana-dashboard.json
+
 ### Visual Studio Code
 
 Similar to Ollama, you can configure Continue to use KubeAi/vLLM. Depending on the model, max tokens or context length will have to be adjusted.

diff --git a/components/ai/kubeai.ts b/components/ai/kubeai.ts
@@ -15,6 +15,7 @@ export class KubeAi extends pulumi.ComponentResource {
 
         const config = new pulumi.Config(name);
         const version = config.get('version');
+        const enableMonitoring = config.getBoolean('enableMonitoring');
         const hostname = config.require('hostname');
         const huggingfaceToken = config.getSecret('huggingfaceToken');
         const models = config.get('models')?.split(',') ?? [];
@@ -43,6 +44,16 @@ export class KubeAi extends pulumi.ComponentResource {
                         ],
                         tls: [{ hosts: [hostname] }],
                     },
+                    metrics: enableMonitoring
+                        ? {
+                              prometheusOperator: {
+                                  vLLMPodMonitor: {
+                                      enabled: true,
+                                      labels: {},
+                                  },
+                              },
+                          }
+                        : undefined,
                     modelAutoscaling: { timeWindow: '30m' },
                     modelServerPods: {
                         // required for NVidia detection

diff --git a/components/monitoring/prometheus.ts b/components/monitoring/prometheus.ts
@@ -96,6 +96,10 @@ export class Prometheus extends pulumi.ComponentResource {
                             hostname: prometheusHostname,
                         },
                         prometheusSpec: {
+                            podMonitorSelectorNilUsesHelmValues: false,
+                            probeSelectorNilUsesHelmValues: false,
+                            ruleSelectorNilUsesHelmValues: false,
+                            serviceMonitorSelectorNilUsesHelmValues: false,
                             storageSpec: {
                                 volumeClaimTemplate: {
                                     spec: {