From c0b9ea1ffef2338e7ba6f5e00555b6da49818b35 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Mon, 4 Oct 2021 12:48:31 +0200
Subject: [PATCH] feat(metrics) add zone selector to Kuma Service dashboard
 (#2865) (#2878)

* feat(metrics) add zone selector to Kuma Service dashboard

Signed-off-by: Mike Beaumont <mjboamail@gmail.com>

* tests(integration) include Kuma Service zone changes in test

Signed-off-by: Mike Beaumont <mjboamail@gmail.com>
(cherry picked from commit e2ee786258157c1977071fe8bfd2e225ab95ae5a)

Co-authored-by: Mike Beaumont <mjboamail@gmail.com>
---
 .../install-metrics.defaults.golden.yaml      | 45 ++++++++++++++-----
 .../install-metrics.no-prometheus.golden.yaml | 45 ++++++++++++++-----
 .../install-metrics.overrides.golden.yaml     | 45 ++++++++++++++-----
 .../k8s/metrics/grafana/kuma-service.json     | 45 ++++++++++++++-----
 4 files changed, 136 insertions(+), 44 deletions(-)

diff --git a/app/kumactl/cmd/install/testdata/install-metrics.defaults.golden.yaml b/app/kumactl/cmd/install/testdata/install-metrics.defaults.golden.yaml
index b26410dbcac8..6c0521e8e627 100644
--- a/app/kumactl/cmd/install/testdata/install-metrics.defaults.golden.yaml
+++ b/app/kumactl/cmd/install/testdata/install-metrics.defaults.golden.yaml
@@ -10617,7 +10617,7 @@ data:
           "pluginVersion": "7.4.3",
           "targets": [
             {
-              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\",envoy_cluster_name=~\"localhost_.*\"}[1m]))",
+              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\",envoy_cluster_name=~\"localhost_.*\"}[1m]))",
               "hide": false,
               "interval": "",
               "legendFormat": "",
@@ -10682,7 +10682,7 @@ data:
           "pluginVersion": "7.4.3",
           "targets": [
             {
-              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\",envoy_cluster_name!~\"localhost_.*\",envoy_cluster_name!=\"kuma_envoy_admin\",envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m]))",
+              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\",envoy_cluster_name!~\"localhost_.*\",envoy_cluster_name!=\"kuma_envoy_admin\",envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m]))",
               "hide": false,
               "interval": "",
               "legendFormat": "",
@@ -10761,21 +10761,21 @@ data:
           "steppedLine": false,
           "targets": [
             {
-              "expr": "max(histogram_quantile(0.99, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
+              "expr": "max(histogram_quantile(0.99, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",kuma_io_zone=~\"$zone\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
               "hide": false,
               "interval": "",
               "legendFormat": "p99",
               "refId": "A"
             },
             {
-              "expr": "max(histogram_quantile(0.95, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
+              "expr": "max(histogram_quantile(0.95, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",kuma_io_zone=~\"$zone\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
               "hide": false,
               "interval": "",
               "legendFormat": "p95",
               "refId": "C"
             },
             {
-              "expr": "max(histogram_quantile(0.50, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
+              "expr": "max(histogram_quantile(0.50, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",kuma_io_zone=~\"$zone\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
               "hide": false,
               "interval": "",
               "legendFormat": "p50",
@@ -10873,14 +10873,14 @@ data:
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\", envoy_cluster_name=~\"localhost_.*\"}[1m]))",
+              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\", envoy_cluster_name=~\"localhost_.*\"}[1m]))",
               "hide": false,
               "interval": "",
               "legendFormat": "Incoming",
               "refId": "C"
             },
             {
-              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\", envoy_cluster_name!~\"localhost_.*\", envoy_cluster_name!=\"kuma_envoy_admin\",envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m]))",
+              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\", envoy_cluster_name!~\"localhost_.*\", envoy_cluster_name!=\"kuma_envoy_admin\",envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m]))",
               "hide": false,
               "interval": "",
               "legendFormat": "Outgoing",
@@ -10978,14 +10978,14 @@ data:
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum(rate(envoy_cluster_external_upstream_rq_xx{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\", envoy_cluster_name=~\"localhost_.*\"}[1m])) by (envoy_response_code_class)",
+              "expr": "sum(rate(envoy_cluster_external_upstream_rq_xx{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\", envoy_cluster_name=~\"localhost_.*\"}[1m])) by (envoy_response_code_class)",
               "hide": false,
               "interval": "",
               "legendFormat": "Incoming {{envoy_response_code_class}}xx",
               "refId": "A"
             },
             {
-              "expr": "sum(rate(envoy_cluster_external_upstream_rq_xx{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\", envoy_cluster_name!~\"localhost_.*\", envoy_cluster_name!=\"kuma_envoy_admin\", envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m])) by (envoy_response_code_class)",
+              "expr": "sum(rate(envoy_cluster_external_upstream_rq_xx{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\", envoy_cluster_name!~\"localhost_.*\", envoy_cluster_name!=\"kuma_envoy_admin\", envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m])) by (envoy_response_code_class)",
               "hide": false,
               "interval": "",
               "legendFormat": "Outgoing {{envoy_response_code_class}}xx",
@@ -11381,6 +11381,29 @@ data:
             "type": "query",
             "useTags": false
           },
+          {
+            "allFormat": "wildcard",
+            "allValue": null,
+            "current": {},
+            "datasource": "Prometheus",
+            "definition": "label_values(envoy_server_live{mesh=\"$mesh\"}, kuma_io_zone)",
+            "hide": 0,
+            "includeAll": true,
+            "label": "Zone",
+            "multi": true,
+            "name": "zone",
+            "options": [],
+            "query": "label_values(envoy_server_live{mesh=\"$mesh\"}, kuma_io_zone)",
+            "refresh": 1,
+            "regex": "",
+            "skipUrlSync": false,
+            "sort": 0,
+            "tagValuesQuery": "",
+            "tags": [],
+            "tagsQuery": "",
+            "type": "query",
+            "useTags": false
+          },
           {
             "allValue": null,
             "current": {
@@ -11389,7 +11412,7 @@ data:
               "value": "backend_kuma-demo_svc_3001"
             },
             "datasource": null,
-            "definition": "label_values(envoy_server_live{mesh=\"$mesh\"}, kuma_io_service)",
+            "definition": "label_values(envoy_server_live{mesh=\"$mesh\",kuma_io_zone=~\"$zone\"}, kuma_io_service)",
             "description": null,
             "error": null,
             "hide": 0,
@@ -11399,7 +11422,7 @@ data:
             "name": "service",
             "options": [],
             "query": {
-              "query": "label_values(envoy_server_live{mesh=\"$mesh\"}, kuma_io_service)",
+              "query": "label_values(envoy_server_live{mesh=\"$mesh\",kuma_io_zone=~\"$zone\"}, kuma_io_service)",
               "refId": "StandardVariableQuery"
             },
             "refresh": 1,
diff --git a/app/kumactl/cmd/install/testdata/install-metrics.no-prometheus.golden.yaml b/app/kumactl/cmd/install/testdata/install-metrics.no-prometheus.golden.yaml
index 1686eca56ec8..338bc2c57645 100644
--- a/app/kumactl/cmd/install/testdata/install-metrics.no-prometheus.golden.yaml
+++ b/app/kumactl/cmd/install/testdata/install-metrics.no-prometheus.golden.yaml
@@ -10240,7 +10240,7 @@ data:
           "pluginVersion": "7.4.3",
           "targets": [
             {
-              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\",envoy_cluster_name=~\"localhost_.*\"}[1m]))",
+              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\",envoy_cluster_name=~\"localhost_.*\"}[1m]))",
               "hide": false,
               "interval": "",
               "legendFormat": "",
@@ -10305,7 +10305,7 @@ data:
           "pluginVersion": "7.4.3",
           "targets": [
             {
-              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\",envoy_cluster_name!~\"localhost_.*\",envoy_cluster_name!=\"kuma_envoy_admin\",envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m]))",
+              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\",envoy_cluster_name!~\"localhost_.*\",envoy_cluster_name!=\"kuma_envoy_admin\",envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m]))",
               "hide": false,
               "interval": "",
               "legendFormat": "",
@@ -10384,21 +10384,21 @@ data:
           "steppedLine": false,
           "targets": [
             {
-              "expr": "max(histogram_quantile(0.99, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
+              "expr": "max(histogram_quantile(0.99, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",kuma_io_zone=~\"$zone\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
               "hide": false,
               "interval": "",
               "legendFormat": "p99",
               "refId": "A"
             },
             {
-              "expr": "max(histogram_quantile(0.95, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
+              "expr": "max(histogram_quantile(0.95, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",kuma_io_zone=~\"$zone\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
               "hide": false,
               "interval": "",
               "legendFormat": "p95",
               "refId": "C"
             },
             {
-              "expr": "max(histogram_quantile(0.50, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
+              "expr": "max(histogram_quantile(0.50, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",kuma_io_zone=~\"$zone\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
               "hide": false,
               "interval": "",
               "legendFormat": "p50",
@@ -10496,14 +10496,14 @@ data:
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\", envoy_cluster_name=~\"localhost_.*\"}[1m]))",
+              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\", envoy_cluster_name=~\"localhost_.*\"}[1m]))",
               "hide": false,
               "interval": "",
               "legendFormat": "Incoming",
               "refId": "C"
             },
             {
-              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\", envoy_cluster_name!~\"localhost_.*\", envoy_cluster_name!=\"kuma_envoy_admin\",envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m]))",
+              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\", envoy_cluster_name!~\"localhost_.*\", envoy_cluster_name!=\"kuma_envoy_admin\",envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m]))",
               "hide": false,
               "interval": "",
               "legendFormat": "Outgoing",
@@ -10601,14 +10601,14 @@ data:
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum(rate(envoy_cluster_external_upstream_rq_xx{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\", envoy_cluster_name=~\"localhost_.*\"}[1m])) by (envoy_response_code_class)",
+              "expr": "sum(rate(envoy_cluster_external_upstream_rq_xx{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\", envoy_cluster_name=~\"localhost_.*\"}[1m])) by (envoy_response_code_class)",
               "hide": false,
               "interval": "",
               "legendFormat": "Incoming {{envoy_response_code_class}}xx",
               "refId": "A"
             },
             {
-              "expr": "sum(rate(envoy_cluster_external_upstream_rq_xx{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\", envoy_cluster_name!~\"localhost_.*\", envoy_cluster_name!=\"kuma_envoy_admin\", envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m])) by (envoy_response_code_class)",
+              "expr": "sum(rate(envoy_cluster_external_upstream_rq_xx{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\", envoy_cluster_name!~\"localhost_.*\", envoy_cluster_name!=\"kuma_envoy_admin\", envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m])) by (envoy_response_code_class)",
               "hide": false,
               "interval": "",
               "legendFormat": "Outgoing {{envoy_response_code_class}}xx",
@@ -11004,6 +11004,29 @@ data:
             "type": "query",
             "useTags": false
           },
+          {
+            "allFormat": "wildcard",
+            "allValue": null,
+            "current": {},
+            "datasource": "Prometheus",
+            "definition": "label_values(envoy_server_live{mesh=\"$mesh\"}, kuma_io_zone)",
+            "hide": 0,
+            "includeAll": true,
+            "label": "Zone",
+            "multi": true,
+            "name": "zone",
+            "options": [],
+            "query": "label_values(envoy_server_live{mesh=\"$mesh\"}, kuma_io_zone)",
+            "refresh": 1,
+            "regex": "",
+            "skipUrlSync": false,
+            "sort": 0,
+            "tagValuesQuery": "",
+            "tags": [],
+            "tagsQuery": "",
+            "type": "query",
+            "useTags": false
+          },
           {
             "allValue": null,
             "current": {
@@ -11012,7 +11035,7 @@ data:
               "value": "backend_kuma-demo_svc_3001"
             },
             "datasource": null,
-            "definition": "label_values(envoy_server_live{mesh=\"$mesh\"}, kuma_io_service)",
+            "definition": "label_values(envoy_server_live{mesh=\"$mesh\",kuma_io_zone=~\"$zone\"}, kuma_io_service)",
             "description": null,
             "error": null,
             "hide": 0,
@@ -11022,7 +11045,7 @@ data:
             "name": "service",
             "options": [],
             "query": {
-              "query": "label_values(envoy_server_live{mesh=\"$mesh\"}, kuma_io_service)",
+              "query": "label_values(envoy_server_live{mesh=\"$mesh\",kuma_io_zone=~\"$zone\"}, kuma_io_service)",
               "refId": "StandardVariableQuery"
             },
             "refresh": 1,
diff --git a/app/kumactl/cmd/install/testdata/install-metrics.overrides.golden.yaml b/app/kumactl/cmd/install/testdata/install-metrics.overrides.golden.yaml
index 67e0516a6d5e..01ba8b657df2 100644
--- a/app/kumactl/cmd/install/testdata/install-metrics.overrides.golden.yaml
+++ b/app/kumactl/cmd/install/testdata/install-metrics.overrides.golden.yaml
@@ -10617,7 +10617,7 @@ data:
           "pluginVersion": "7.4.3",
           "targets": [
             {
-              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\",envoy_cluster_name=~\"localhost_.*\"}[1m]))",
+              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\",envoy_cluster_name=~\"localhost_.*\"}[1m]))",
               "hide": false,
               "interval": "",
               "legendFormat": "",
@@ -10682,7 +10682,7 @@ data:
           "pluginVersion": "7.4.3",
           "targets": [
             {
-              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\",envoy_cluster_name!~\"localhost_.*\",envoy_cluster_name!=\"kuma_envoy_admin\",envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m]))",
+              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\",envoy_cluster_name!~\"localhost_.*\",envoy_cluster_name!=\"kuma_envoy_admin\",envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m]))",
               "hide": false,
               "interval": "",
               "legendFormat": "",
@@ -10761,21 +10761,21 @@ data:
           "steppedLine": false,
           "targets": [
             {
-              "expr": "max(histogram_quantile(0.99, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
+              "expr": "max(histogram_quantile(0.99, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",kuma_io_zone=~\"$zone\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
               "hide": false,
               "interval": "",
               "legendFormat": "p99",
               "refId": "A"
             },
             {
-              "expr": "max(histogram_quantile(0.95, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
+              "expr": "max(histogram_quantile(0.95, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",kuma_io_zone=~\"$zone\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
               "hide": false,
               "interval": "",
               "legendFormat": "p95",
               "refId": "C"
             },
             {
-              "expr": "max(histogram_quantile(0.50, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
+              "expr": "max(histogram_quantile(0.50, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",kuma_io_zone=~\"$zone\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
               "hide": false,
               "interval": "",
               "legendFormat": "p50",
@@ -10873,14 +10873,14 @@ data:
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\", envoy_cluster_name=~\"localhost_.*\"}[1m]))",
+              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\", envoy_cluster_name=~\"localhost_.*\"}[1m]))",
               "hide": false,
               "interval": "",
               "legendFormat": "Incoming",
               "refId": "C"
             },
             {
-              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\", envoy_cluster_name!~\"localhost_.*\", envoy_cluster_name!=\"kuma_envoy_admin\",envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m]))",
+              "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\", envoy_cluster_name!~\"localhost_.*\", envoy_cluster_name!=\"kuma_envoy_admin\",envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m]))",
               "hide": false,
               "interval": "",
               "legendFormat": "Outgoing",
@@ -10978,14 +10978,14 @@ data:
           "steppedLine": false,
           "targets": [
             {
-              "expr": "sum(rate(envoy_cluster_external_upstream_rq_xx{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\", envoy_cluster_name=~\"localhost_.*\"}[1m])) by (envoy_response_code_class)",
+              "expr": "sum(rate(envoy_cluster_external_upstream_rq_xx{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\", envoy_cluster_name=~\"localhost_.*\"}[1m])) by (envoy_response_code_class)",
               "hide": false,
               "interval": "",
               "legendFormat": "Incoming {{envoy_response_code_class}}xx",
               "refId": "A"
             },
             {
-              "expr": "sum(rate(envoy_cluster_external_upstream_rq_xx{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\", envoy_cluster_name!~\"localhost_.*\", envoy_cluster_name!=\"kuma_envoy_admin\", envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m])) by (envoy_response_code_class)",
+              "expr": "sum(rate(envoy_cluster_external_upstream_rq_xx{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\", envoy_cluster_name!~\"localhost_.*\", envoy_cluster_name!=\"kuma_envoy_admin\", envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m])) by (envoy_response_code_class)",
               "hide": false,
               "interval": "",
               "legendFormat": "Outgoing {{envoy_response_code_class}}xx",
@@ -11381,6 +11381,29 @@ data:
             "type": "query",
             "useTags": false
           },
+          {
+            "allFormat": "wildcard",
+            "allValue": null,
+            "current": {},
+            "datasource": "Prometheus",
+            "definition": "label_values(envoy_server_live{mesh=\"$mesh\"}, kuma_io_zone)",
+            "hide": 0,
+            "includeAll": true,
+            "label": "Zone",
+            "multi": true,
+            "name": "zone",
+            "options": [],
+            "query": "label_values(envoy_server_live{mesh=\"$mesh\"}, kuma_io_zone)",
+            "refresh": 1,
+            "regex": "",
+            "skipUrlSync": false,
+            "sort": 0,
+            "tagValuesQuery": "",
+            "tags": [],
+            "tagsQuery": "",
+            "type": "query",
+            "useTags": false
+          },
           {
             "allValue": null,
             "current": {
@@ -11389,7 +11412,7 @@ data:
               "value": "backend_kuma-demo_svc_3001"
             },
             "datasource": null,
-            "definition": "label_values(envoy_server_live{mesh=\"$mesh\"}, kuma_io_service)",
+            "definition": "label_values(envoy_server_live{mesh=\"$mesh\",kuma_io_zone=~\"$zone\"}, kuma_io_service)",
             "description": null,
             "error": null,
             "hide": 0,
@@ -11399,7 +11422,7 @@ data:
             "name": "service",
             "options": [],
             "query": {
-              "query": "label_values(envoy_server_live{mesh=\"$mesh\"}, kuma_io_service)",
+              "query": "label_values(envoy_server_live{mesh=\"$mesh\",kuma_io_zone=~\"$zone\"}, kuma_io_service)",
               "refId": "StandardVariableQuery"
             },
             "refresh": 1,
diff --git a/app/kumactl/data/install/k8s/metrics/grafana/kuma-service.json b/app/kumactl/data/install/k8s/metrics/grafana/kuma-service.json
index fb8306585f9c..d7146aa734b7 100644
--- a/app/kumactl/data/install/k8s/metrics/grafana/kuma-service.json
+++ b/app/kumactl/data/install/k8s/metrics/grafana/kuma-service.json
@@ -213,7 +213,7 @@
       "pluginVersion": "7.4.3",
       "targets": [
         {
-          "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\",envoy_cluster_name=~\"localhost_.*\"}[1m]))",
+          "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\",envoy_cluster_name=~\"localhost_.*\"}[1m]))",
           "hide": false,
           "interval": "",
           "legendFormat": "",
@@ -278,7 +278,7 @@
       "pluginVersion": "7.4.3",
       "targets": [
         {
-          "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\",envoy_cluster_name!~\"localhost_.*\",envoy_cluster_name!=\"kuma_envoy_admin\",envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m]))",
+          "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\",envoy_cluster_name!~\"localhost_.*\",envoy_cluster_name!=\"kuma_envoy_admin\",envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m]))",
           "hide": false,
           "interval": "",
           "legendFormat": "",
@@ -357,21 +357,21 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "max(histogram_quantile(0.99, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
+          "expr": "max(histogram_quantile(0.99, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",kuma_io_zone=~\"$zone\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
           "hide": false,
           "interval": "",
           "legendFormat": "p99",
           "refId": "A"
         },
         {
-          "expr": "max(histogram_quantile(0.95, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
+          "expr": "max(histogram_quantile(0.95, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",kuma_io_zone=~\"$zone\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
           "hide": false,
           "interval": "",
           "legendFormat": "p95",
           "refId": "C"
         },
         {
-          "expr": "max(histogram_quantile(0.50, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
+          "expr": "max(histogram_quantile(0.50, rate(envoy_cluster_upstream_rq_time_bucket{kuma_io_services=~\".*$service.*\",kuma_io_zone=~\"$zone\",mesh=\"$mesh\",envoy_cluster_name=~\"localhost_.*\"}[1m])))",
           "hide": false,
           "interval": "",
           "legendFormat": "p50",
@@ -469,14 +469,14 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\", envoy_cluster_name=~\"localhost_.*\"}[1m]))",
+          "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\", envoy_cluster_name=~\"localhost_.*\"}[1m]))",
           "hide": false,
           "interval": "",
           "legendFormat": "Incoming",
           "refId": "C"
         },
         {
-          "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\", envoy_cluster_name!~\"localhost_.*\", envoy_cluster_name!=\"kuma_envoy_admin\",envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m]))",
+          "expr": "sum(rate(envoy_cluster_upstream_rq_total{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\", envoy_cluster_name!~\"localhost_.*\", envoy_cluster_name!=\"kuma_envoy_admin\",envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m]))",
           "hide": false,
           "interval": "",
           "legendFormat": "Outgoing",
@@ -574,14 +574,14 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "sum(rate(envoy_cluster_external_upstream_rq_xx{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\", envoy_cluster_name=~\"localhost_.*\"}[1m])) by (envoy_response_code_class)",
+          "expr": "sum(rate(envoy_cluster_external_upstream_rq_xx{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\", envoy_cluster_name=~\"localhost_.*\"}[1m])) by (envoy_response_code_class)",
           "hide": false,
           "interval": "",
           "legendFormat": "Incoming {{envoy_response_code_class}}xx",
           "refId": "A"
         },
         {
-          "expr": "sum(rate(envoy_cluster_external_upstream_rq_xx{mesh=\"$mesh\",kuma_io_services=~\".*$service.*\", envoy_cluster_name!~\"localhost_.*\", envoy_cluster_name!=\"kuma_envoy_admin\", envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m])) by (envoy_response_code_class)",
+          "expr": "sum(rate(envoy_cluster_external_upstream_rq_xx{mesh=\"$mesh\",kuma_io_zone=~\"$zone\",kuma_io_services=~\".*$service.*\", envoy_cluster_name!~\"localhost_.*\", envoy_cluster_name!=\"kuma_envoy_admin\", envoy_cluster_name!=\"kuma_metrics_hijacker\"}[1m])) by (envoy_response_code_class)",
           "hide": false,
           "interval": "",
           "legendFormat": "Outgoing {{envoy_response_code_class}}xx",
@@ -977,6 +977,29 @@
         "type": "query",
         "useTags": false
       },
+      {
+        "allFormat": "wildcard",
+        "allValue": null,
+        "current": {},
+        "datasource": "Prometheus",
+        "definition": "label_values(envoy_server_live{mesh=\"$mesh\"}, kuma_io_zone)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "Zone",
+        "multi": true,
+        "name": "zone",
+        "options": [],
+        "query": "label_values(envoy_server_live{mesh=\"$mesh\"}, kuma_io_zone)",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      },
       {
         "allValue": null,
         "current": {
@@ -985,7 +1008,7 @@
           "value": "backend_kuma-demo_svc_3001"
         },
         "datasource": null,
-        "definition": "label_values(envoy_server_live{mesh=\"$mesh\"}, kuma_io_service)",
+        "definition": "label_values(envoy_server_live{mesh=\"$mesh\",kuma_io_zone=~\"$zone\"}, kuma_io_service)",
         "description": null,
         "error": null,
         "hide": 0,
@@ -995,7 +1018,7 @@
         "name": "service",
         "options": [],
         "query": {
-          "query": "label_values(envoy_server_live{mesh=\"$mesh\"}, kuma_io_service)",
+          "query": "label_values(envoy_server_live{mesh=\"$mesh\",kuma_io_zone=~\"$zone\"}, kuma_io_service)",
           "refId": "StandardVariableQuery"
         },
         "refresh": 1,