Add switch to allow enabling overrides-exporter ring (#3995)

* add switch for overrides-exporter ring * second iteration * make build-jsonnet-tests * do not add ring flags when the ring is not enabled * make build-jsonnet-tests * Add changelog
grafana · Jan 20, 2023 · efd72bb · efd72bb
1 parent f143ade
commit efd72bb
Show file tree

Hide file tree

Showing 8 changed files with 62 additions and 4 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -86,6 +86,7 @@
 * [ENHANCEMENT] Update `rollout-operator` to `v0.2.0`. #3624
 * [ENHANCEMENT] Add `user_24M` and `user_32M` classes to operations config. #3367
 * [ENHANCEMENT] Update memcached image from `memcached:1.6.16-alpine` to `memcached:1.6.17-alpine`. #3914
+* [ENHANCEMENT] Allow configuring the ring for overrides-exporter. #3995
 * [BUGFIX] Apply ingesters and store-gateways per-zone CLI flags overrides to read-write deployment mode too. #3766
 * [BUGFIX] Apply overrides-exporter CLI flags to mimir-backend when running Mimir in read-write deployment mode. #3790
 * [BUGFIX] Fixed `mimir-write` and `mimir-read` Kubernetes service to correctly balance requests among pods. #3855 #3864 #3906

diff --git a/operations/mimir-tests/test-deployment-mode-migration-generated.yaml b/operations/mimir-tests/test-deployment-mode-migration-generated.yaml
@@ -2213,6 +2213,10 @@ spec:
         - -ingester.ring.zone-awareness-enabled=true
         - -memberlist.bind-port=7946
         - -memberlist.join=dns+gossip-ring.default.svc.cluster.local:7946
+        - -overrides-exporter.ring.enabled=true
+        - -overrides-exporter.ring.prefix=
+        - -overrides-exporter.ring.store=memberlist
+        - -overrides-exporter.ring.wait-stability-min-duration=1m
         - -querier.max-partial-query-length=768h
         - -query-scheduler.max-outstanding-requests-per-tenant=100
         - -query-scheduler.max-used-instances=2
@@ -2385,6 +2389,10 @@ spec:
         - -ingester.ring.zone-awareness-enabled=true
         - -memberlist.bind-port=7946
         - -memberlist.join=dns+gossip-ring.default.svc.cluster.local:7946
+        - -overrides-exporter.ring.enabled=true
+        - -overrides-exporter.ring.prefix=
+        - -overrides-exporter.ring.store=memberlist
+        - -overrides-exporter.ring.wait-stability-min-duration=1m
         - -querier.max-partial-query-length=768h
         - -query-scheduler.max-outstanding-requests-per-tenant=100
         - -query-scheduler.max-used-instances=2
@@ -2557,6 +2565,10 @@ spec:
         - -ingester.ring.zone-awareness-enabled=true
         - -memberlist.bind-port=7946
         - -memberlist.join=dns+gossip-ring.default.svc.cluster.local:7946
+        - -overrides-exporter.ring.enabled=true
+        - -overrides-exporter.ring.prefix=
+        - -overrides-exporter.ring.store=memberlist
+        - -overrides-exporter.ring.wait-stability-min-duration=1m
         - -querier.max-partial-query-length=768h
         - -query-scheduler.max-outstanding-requests-per-tenant=100
         - -query-scheduler.max-used-instances=2

diff --git a/operations/mimir-tests/test-read-write-deployment-mode-s3-autoscaled-generated.yaml b/operations/mimir-tests/test-read-write-deployment-mode-s3-autoscaled-generated.yaml
@@ -865,6 +865,10 @@ spec:
         - -ingester.ring.zone-awareness-enabled=true
         - -memberlist.bind-port=7946
         - -memberlist.join=dns+gossip-ring.default.svc.cluster.local:7946
+        - -overrides-exporter.ring.enabled=true
+        - -overrides-exporter.ring.prefix=
+        - -overrides-exporter.ring.store=memberlist
+        - -overrides-exporter.ring.wait-stability-min-duration=1m
         - -querier.max-partial-query-length=768h
         - -query-scheduler.max-outstanding-requests-per-tenant=100
         - -query-scheduler.max-used-instances=2
@@ -1038,6 +1042,10 @@ spec:
         - -ingester.ring.zone-awareness-enabled=true
         - -memberlist.bind-port=7946
         - -memberlist.join=dns+gossip-ring.default.svc.cluster.local:7946
+        - -overrides-exporter.ring.enabled=true
+        - -overrides-exporter.ring.prefix=
+        - -overrides-exporter.ring.store=memberlist
+        - -overrides-exporter.ring.wait-stability-min-duration=1m
         - -querier.max-partial-query-length=768h
         - -query-scheduler.max-outstanding-requests-per-tenant=100
         - -query-scheduler.max-used-instances=2
@@ -1211,6 +1219,10 @@ spec:
         - -ingester.ring.zone-awareness-enabled=true
         - -memberlist.bind-port=7946
         - -memberlist.join=dns+gossip-ring.default.svc.cluster.local:7946
+        - -overrides-exporter.ring.enabled=true
+        - -overrides-exporter.ring.prefix=
+        - -overrides-exporter.ring.store=memberlist
+        - -overrides-exporter.ring.wait-stability-min-duration=1m
         - -querier.max-partial-query-length=768h
         - -query-scheduler.max-outstanding-requests-per-tenant=100
         - -query-scheduler.max-used-instances=2

diff --git a/operations/mimir-tests/test-read-write-deployment-mode-s3-generated.yaml b/operations/mimir-tests/test-read-write-deployment-mode-s3-generated.yaml
@@ -866,6 +866,10 @@ spec:
         - -ingester.ring.zone-awareness-enabled=true
         - -memberlist.bind-port=7946
         - -memberlist.join=dns+gossip-ring.default.svc.cluster.local:7946
+        - -overrides-exporter.ring.enabled=true
+        - -overrides-exporter.ring.prefix=
+        - -overrides-exporter.ring.store=memberlist
+        - -overrides-exporter.ring.wait-stability-min-duration=1m
         - -querier.max-partial-query-length=768h
         - -query-scheduler.max-outstanding-requests-per-tenant=100
         - -query-scheduler.max-used-instances=2
@@ -1039,6 +1043,10 @@ spec:
         - -ingester.ring.zone-awareness-enabled=true
         - -memberlist.bind-port=7946
         - -memberlist.join=dns+gossip-ring.default.svc.cluster.local:7946
+        - -overrides-exporter.ring.enabled=true
+        - -overrides-exporter.ring.prefix=
+        - -overrides-exporter.ring.store=memberlist
+        - -overrides-exporter.ring.wait-stability-min-duration=1m
         - -querier.max-partial-query-length=768h
         - -query-scheduler.max-outstanding-requests-per-tenant=100
         - -query-scheduler.max-used-instances=2
@@ -1212,6 +1220,10 @@ spec:
         - -ingester.ring.zone-awareness-enabled=true
         - -memberlist.bind-port=7946
         - -memberlist.join=dns+gossip-ring.default.svc.cluster.local:7946
+        - -overrides-exporter.ring.enabled=true
+        - -overrides-exporter.ring.prefix=
+        - -overrides-exporter.ring.store=memberlist
+        - -overrides-exporter.ring.wait-stability-min-duration=1m
         - -querier.max-partial-query-length=768h
         - -query-scheduler.max-outstanding-requests-per-tenant=100
         - -query-scheduler.max-used-instances=2

diff --git a/operations/mimir/config.libsonnet b/operations/mimir/config.libsonnet
@@ -218,6 +218,18 @@
       else
         querySchedulerRingConfig,
 
+    overridesExporterRingConfig:
+      if !$._config.overrides_exporter_ring_enabled then
+        {}
+      else
+        {
+          'overrides-exporter.ring.enabled': true,
+          'overrides-exporter.ring.store': 'consul',
+          'overrides-exporter.ring.consul.hostname': 'consul.%s.svc.cluster.local:8500' % $._config.namespace,
+          'overrides-exporter.ring.prefix': '',
+          'overrides-exporter.ring.wait-stability-min-duration': '1m',
+        },
+
     ruler_enabled: false,
     ruler_storage_backend: $._config.storage_backend,
     ruler_storage_bucket_name: error 'must specify the ruler storage bucket name',

diff --git a/operations/mimir/memberlist.libsonnet b/operations/mimir/memberlist.libsonnet
@@ -46,6 +46,9 @@
 
     queryBlocksStorageConfig+:: if !$._config.memberlist_ring_enabled then {} else (setupGossipRing('store-gateway.sharding-ring.store', 'store-gateway.sharding-ring.consul.hostname', 'store-gateway.sharding-ring.multi') + $._config.memberlistConfig),
 
+    overridesExporterRingConfig+: if !$._config.memberlist_ring_enabled || !$._config.overrides_exporter_ring_enabled then {}
+    else (setupGossipRing('overrides-exporter.ring.store', 'overrides-exporter.ring.consul.hostname', 'overrides-exporter.ring.multi') + $._config.memberlistConfig),
+
     querySchedulerRingClientConfig+: if !queryFrontendMemberlistEnabled then {} else (setupGossipRing('query-scheduler.ring.store', 'query-scheduler.ring.consul.hostname', 'query-scheduler.ring.multi') + $._config.memberlistConfig),
     querySchedulerRingLifecyclerConfig+: if !querySchedulerMemberlistEnabled then {} else (setupGossipRing('query-scheduler.ring.store', 'query-scheduler.ring.consul.hostname', 'query-scheduler.ring.multi') + $._config.memberlistConfig),
 

diff --git a/operations/mimir/overrides-exporter.libsonnet b/operations/mimir/overrides-exporter.libsonnet
@@ -6,6 +6,7 @@
 
   _config+: {
     overrides_exporter_enabled: false,
+    overrides_exporter_ring_enabled: false,
 
     // overrides exporter can also make the configured presets available, this
     // list references entries within $._config.overrides
@@ -24,11 +25,15 @@
   local containerPort = $.core.v1.containerPort,
   overrides_exporter_port:: containerPort.newNamed(name='http-metrics', containerPort=$._config.server_http_port),
 
-  overrides_exporter_args:: {
-    target: 'overrides-exporter',
+  overrides_exporter_args::
+    $._config.limitsConfig +
+    $._config.overridesExporterRingConfig +
+    $.mimirRuntimeConfigFile +
+    {
+      target: 'overrides-exporter',
 
-    'server.http-listen-port': $._config.server_http_port,
-  } + $._config.limitsConfig + $.mimirRuntimeConfigFile,
+      'server.http-listen-port': $._config.server_http_port,
+    },
 
   local container = $.core.v1.container,
   overrides_exporter_container::

diff --git a/operations/mimir/read-write-deployment/config.libsonnet b/operations/mimir/read-write-deployment/config.libsonnet
@@ -18,6 +18,7 @@
 
     // Overrides-exporter is part of the backend component in the Mimir read-write deployment mode.
     overrides_exporter_enabled: if $._config.is_read_write_deployment_mode then false else super.overrides_exporter_enabled,
+    overrides_exporter_ring_enabled: if $._config.is_read_write_deployment_mode then true else super.overrides_exporter_enabled,
   },
 
   // Mimir read-write deployment mode makes some strong assumptions about what must enabled enabled and disabled.