Single ServiceMonitor for store shards

The Prometheus operator creates 1 job per endpoint on each
ServiceMonitor object it discovers. When using the thanos-store hashmod
sharding feature of kube-thanos, 1 ServiceMonitor was created per store
shard.  This resulted in 1 prometheus job per shard, with names like
"thanos-store-0", "thanos-store-1".

This commit changes this behavior to instead create only 1
ServiceMonitor, with a Service label selector that selects all Services
for the shards.

Signed-off-by: Craig Furman <craig.furman89@gmail.com>

CHANGELOG.md

@@ -15,15 +15,15 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 
 ### Changed
 
--
+- [#188](https://github.com/thanos-io/kube-thanos/pull/188) Single ServiceMonitor for store shards
 
 ### Added
 
 -
 
 ### Fixed
 
-- [#185](https://github.com/thanos-io/kube-thanos/pull/185)A query-frontend, store: make cache types case insensitive
+- [#185](https://github.com/thanos-io/kube-thanos/pull/185) query-frontend, store: make cache types case insensitive
 
 ## [v0.17.0](https://github.com/thanos-io/kube-thanos/tree/v0.17.0) (2020-12-08)
 

all.jsonnet

@@ -193,7 +193,7 @@ local finalQ = t.query(q.config {
     'dnssrv+_grpc._tcp.%s.%s.svc.cluster.local' % [service.metadata.name, service.metadata.namespace]
     for service in [re.service, ru.service, s.service] +
                    [rcvs[hashring].service for hashring in std.objectFields(rcvs)] +
-                   [strs[shard].service for shard in std.objectFields(strs)]
+                   [strs.shards[shard].service for shard in std.objectFields(strs.shards)]
   ],
 });
 
@@ -211,8 +211,11 @@ local finalQ = t.query(q.config {
   if rcvs[hashring][name] != null
 } +
 {
-  ['store-' + shard + '-' + name]: strs[shard][name]
-  for shard in std.objectFields(strs)
-  for name in std.objectFields(strs[shard])
-  if strs[shard][name] != null
+  ['store-' + shard + '-' + name]: strs.shards[shard][name]
+  for shard in std.objectFields(strs.shards)
+  for name in std.objectFields(strs.shards[shard])
+  if strs.shards[shard][name] != null
+} +
+{
+  'store-shards-serviceMonitor': strs.serviceMonitor,
 }

examples/all/manifests/store-shard0-serviceMonitor.yaml → examples/all/manifests/store-shards-serviceMonitor.yaml

@@ -3,11 +3,10 @@ kind: ServiceMonitor
 metadata:
   labels:
     app.kubernetes.io/component: object-store-gateway
-    app.kubernetes.io/instance: thanos-store-0
+    app.kubernetes.io/instance: thanos-store
     app.kubernetes.io/name: thanos-store
     app.kubernetes.io/version: v0.17.2
-    store.observatorium.io/shard: shard-0
-  name: thanos-store-0
+  name: thanos-store
   namespace: thanos
 spec:
   endpoints:
@@ -18,9 +17,12 @@ spec:
       - namespace
       - pod
       targetLabel: instance
+    - regex: shard\-(\d+)
+      replacement: $1
+      sourceLabels:
+      - __meta_kubernetes_service_label_store_observatorium_io_shard
+      targetLabel: shard
   selector:
     matchLabels:
       app.kubernetes.io/component: object-store-gateway
-      app.kubernetes.io/instance: thanos-store-0
       app.kubernetes.io/name: thanos-store
-      store.observatorium.io/shard: shard-0

jsonnet/kube-thanos/kube-thanos-store-shards.libsonnet

@@ -16,35 +16,76 @@ function(params)
   assert std.isNumber(config.shards) && config.shards >= 0 : 'thanos store shards has to be number >= 0';
 
   { config:: config } + {
-    ['shard' + i]: store(config {
-      name+: '-%d' % i,
-      commonLabels+:: { 'store.observatorium.io/shard': 'shard-' + i },
-    }) {
-      statefulSet+: {
-        spec+: {
-          template+: {
-            spec+: {
-              containers: [
-                if c.name == 'thanos-store' then c {
-                  args+: [
-                    |||
-                      --selector.relabel-config=
-                        - action: hashmod
-                          source_labels: ["__block_id"]
-                          target_label: shard
-                          modulus: %d
-                        - action: keep
-                          source_labels: ["shard"]
-                          regex: %d
-                    ||| % [config.shards, i],
-                  ],
-                } else c
-                for c in super.containers
-              ],
+    shards: {
+      ['shard' + i]: store(config {
+        name+: '-%d' % i,
+        commonLabels+:: { 'store.observatorium.io/shard': 'shard-' + i },
+      }) {
+        statefulSet+: {
+          spec+: {
+            template+: {
+              spec+: {
+                containers: [
+                  if c.name == 'thanos-store' then c {
+                    args+: [
+                      |||
+                        --selector.relabel-config=
+                          - action: hashmod
+                            source_labels: ["__block_id"]
+                            target_label: shard
+                            modulus: %d
+                          - action: keep
+                            source_labels: ["shard"]
+                            regex: %d
+                      ||| % [config.shards, i],
+                    ],
+                  } else c
+                  for c in super.containers
+                ],
+              },
             },
           },
         },
+
+        serviceMonitor: null,
+      }
+      for i in std.range(0, config.shards - 1)
+    },
+  } + {
+    serviceMonitor: if config.serviceMonitor == true then {
+      apiVersion: 'monitoring.coreos.com/v1',
+      kind: 'ServiceMonitor',
+      metadata+: {
+        name: config.name,
+        namespace: config.namespace,
+        labels: config.commonLabels,
+      },
+      spec: {
+        selector: {
+          matchLabels: {
+            [key]: config.podLabelSelector[key]
+            for key in std.objectFields(config.podLabelSelector)
+            if key != 'app.kubernetes.io/instance'
+          },
+        },
+        endpoints: [
+          {
+            port: 'http',
+            relabelings: [
+              {
+                sourceLabels: ['namespace', 'pod'],
+                separator: '/',
+                targetLabel: 'instance',
+              },
+              {
+                sourceLabels: ['__meta_kubernetes_service_label_store_observatorium_io_shard'],
+                regex: 'shard\\-(\\d+)',
+                replacement: '$1',
+                targetLabel: 'shard',
+              },
+            ],
+          },
+        ],
       },
-    }
-    for i in std.range(0, config.shards - 1)
+    },
   }