Make collection of Prometheus data be controlled by an annotation.

config/constants.go

@@ -66,6 +66,10 @@ const (
 	// DriverPortEnv specifies the name of the env variable that contains driver port.
 	DriverPortEnv = "DRIVER_PORT"
 
+	// EnablePrometheusEnv specifies the name of the env variable that indicates
+	// if the collection of Prometheus data is enabled.
+	EnablePrometheusEnv = "ENABLE_PROMETHEUS"
+
 	// PoolLabel is the key for a label which will have the name of a pool as
 	// the value.
 	PoolLabel = "pool"

config/samples/templates/psm/cxx_example_loadtest_proxied.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     scenario: cpp_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: proxied
   labels:

config/samples/templates/psm/cxx_example_loadtest_proxyless.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     scenario: cpp_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: proxyless
   labels:

config/samples/templates/psm/go_example_loadtest_proxied.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     scenario: go_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: proxied
   labels:

config/samples/templates/psm/go_example_loadtest_proxyless.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     scenario: go_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: proxyless
   labels:

config/samples/templates/psm/java_example_loadtest_proxied.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     scenario: java_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: proxied
   labels:

config/samples/templates/psm/java_example_loadtest_proxyless.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     scenario: java_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: proxyless
   labels:

config/samples/templates/psm/node_example_loadtest_proxied.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     scenario: node_to_node_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: proxied
   labels:

config/samples/templates/psm/node_example_loadtest_proxyless.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     scenario: node_to_node_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: proxyless
   labels:

config/samples/templates/psm/php7_example_loadtest_proxied.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     scenario: php7_protobuf_php_extension_to_cpp_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: proxied
   labels:

config/samples/templates/psm/php7_example_loadtest_proxyless.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     scenario: php7_protobuf_php_extension_to_cpp_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: proxyless
   labels:

config/samples/templates/psm/php7_protobuf_c_example_loadtest_proxied.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     scenario: php7_protobuf_c_extension_to_cpp_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: proxied
   labels:

config/samples/templates/psm/php7_protobuf_c_example_loadtest_proxyless.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     scenario: php7_protobuf_c_extension_to_cpp_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: proxyless
   labels:

config/samples/templates/psm/prebuilt/cxx_example_loadtest_proxied_with_prebuilt_workers.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     pool: ${workers_pool}
     scenario: cpp_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: prebuilt-proxied

config/samples/templates/psm/prebuilt/cxx_example_loadtest_proxyless_with_prebuilt_workers.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     pool: ${workers_pool}
     scenario: cpp_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: prebuilt-proxyless

config/samples/templates/psm/prebuilt/go_example_loadtest_proxied_with_prebuilt_workers.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     pool: ${workers_pool}
     scenario: go_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: prebuilt-proxied

config/samples/templates/psm/prebuilt/go_example_loadtest_proxyless_with_prebuilt_workers.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     pool: ${workers_pool}
     scenario: go_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: prebuilt-proxyless

config/samples/templates/psm/prebuilt/java_example_loadtest_proxied_with_prebuilt_workers.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     pool: ${workers_pool}
     scenario: java_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: prebuilt-proxied

config/samples/templates/psm/prebuilt/java_example_loadtest_proxyless_with_prebuilt_workers.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     pool: ${workers_pool}
     scenario: java_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: prebuilt-proxyless

config/samples/templates/psm/prebuilt/node_example_loadtest_proxied_with_prebuilt_workers.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     pool: ${workers_pool}
     scenario: node_to_node_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: prebuilt-proxied

config/samples/templates/psm/prebuilt/node_example_loadtest_proxyless_with_prebuilt_workers.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     pool: ${workers_pool}
     scenario: node_to_node_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: prebuilt-proxyless

config/samples/templates/psm/prebuilt/php7_example_loadtest_proxied_with_prebuilt_workers.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     pool: ${workers_pool}
     scenario: php7_protobuf_php_extension_to_cpp_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: prebuilt-proxied

config/samples/templates/psm/prebuilt/php7_example_loadtest_proxyless_with_prebuilt_workers.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     pool: ${workers_pool}
     scenario: php7_protobuf_php_extension_to_cpp_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: prebuilt-proxyless

config/samples/templates/psm/prebuilt/php7_protobuf_c_example_loadtest_proxied_with_prebuilt_workers.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     pool: ${workers_pool}
     scenario: php7_protobuf_c_extension_to_cpp_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: prebuilt-proxied

config/samples/templates/psm/prebuilt/php7_protobuf_c_example_loadtest_proxyless_with_prebuilt_workers.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     pool: ${workers_pool}
     scenario: php7_protobuf_c_extension_to_cpp_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: prebuilt-proxyless

config/samples/templates/psm/prebuilt/python_asyncio_example_loadtest_proxied_with_prebuilt_workers.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     pool: ${workers_pool}
     scenario: python_asyncio_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: prebuilt-proxied

config/samples/templates/psm/prebuilt/python_asyncio_example_loadtest_proxyless_with_prebuilt_workers.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     pool: ${workers_pool}
     scenario: python_asyncio_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: prebuilt-proxyless

config/samples/templates/psm/prebuilt/python_example_loadtest_proxied_with_prebuilt_workers.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     pool: ${workers_pool}
     scenario: python_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: prebuilt-proxied

config/samples/templates/psm/prebuilt/python_example_loadtest_proxyless_with_prebuilt_workers.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     pool: ${workers_pool}
     scenario: python_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: prebuilt-proxyless

config/samples/templates/psm/python_asyncio_example_loadtest_proxied.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     scenario: python_asyncio_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: proxied
   labels:

config/samples/templates/psm/python_asyncio_example_loadtest_proxyless.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     scenario: python_asyncio_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: proxyless
   labels:

config/samples/templates/psm/python_example_loadtest_proxied.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     scenario: python_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: proxied
   labels:

config/samples/templates/psm/python_example_loadtest_proxyless.yaml

@@ -5,6 +5,7 @@ apiVersion: e2etest.grpc.io/v1
 kind: LoadTest
 metadata:
   annotations:
+    enablePrometheus: 'true'
     scenario: python_protobuf_async_unary_5000rpcs_1KB_psm_8channels_16threads_5000load
     uniquifier: proxyless
   labels:

containers/runtime/driver/Dockerfile

@@ -63,6 +63,7 @@ RUN apt-get update && apt-get install -y \
   pkg-config \
   gnupg \
   apt-transport-https \
+  dnsutils \
   ca-certificates
 
 RUN apt-get update && apt-get install -y \

containers/runtime/driver/run.sh

@@ -35,11 +35,13 @@ if [ -n "${BQ_RESULT_TABLE}" ]; then
   fi
   if [ -r "${NODE_INFO_OUTPUT_FILE}" ]; then
     cp "${NODE_INFO_OUTPUT_FILE}" node_info.json
-    if [ -n "${SERVER_TARGET_OVERRIDE}" ]; then
-      python3 /src/code/tools/run_tests/performance/prometheus.py \
-        --url=http://prometheus.test-infra-system.svc.cluster.local:9090 \
-        --pod_type=clients --container_name=main \
-        --container_name=sidecar
+    if [ -z "${SERVER_TARGET_OVERRIDE}" ] || [ -z "${ENABLE_PROMETHEUS}" ]; then
+      if  [ "$(dig +short -t srv prometheus.test-infra-system.svc.cluster.local)" ]; then
+        python3 /src/code/tools/run_tests/performance/prometheus.py \
+          --url=http://prometheus.test-infra-system.svc.cluster.local:9090 \
+          --pod_type=clients --container_name=main \
+          --container_name=sidecar
+      fi
     fi
   fi
   python3 /src/code/tools/run_tests/performance/bq_upload_result.py --bq_result_table="${BQ_RESULT_TABLE}"

kubehelpers/psm.go

@@ -20,7 +20,7 @@ import (
 	"github.com/grpc/test-infra/config"
 )
 
-//IsPSMTest checks if a given LoadTest is a (proxied or proxyless) service
+// IsPSMTest checks if a given LoadTest is a (proxied or proxyless) service
 // mesh test. This test must be performed after validating the client specs.
 func IsPSMTest(clients *[]grpcv1.Client) bool {
 	for _, c := range *clients {

podbuilder/podbuilder.go

@@ -18,6 +18,7 @@ package podbuilder
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
@@ -240,6 +241,14 @@ func (pb *PodBuilder) PodForDriver(driver *grpcv1.Driver) (*corev1.Pod, error) {
 		}
 	}
 
+	enablePrometheus, ok := pb.test.Annotations["enablePrometheus"]
+	if ok && strings.ToLower(enablePrometheus) == "true" {
+		runContainer.Env = append(runContainer.Env,
+			corev1.EnvVar{
+				Name:  config.EnablePrometheusEnv,
+				Value: "true"})
+	}
+
 	return pod, nil
 }