Merge pull request grafana/cortex-jsonnet#287 from grafana/add-functi…

…on-to-create-compactor-statefulset Add function to customize compactor statefulset
grafana · Apr 2, 2021 · 2715796 · 2715796
2 parents f1fb713 + d7fbc23
commit 2715796
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 20 deletions.
diff --git a/jsonnet/mimir-mixin/alerts/compactor.libsonnet b/jsonnet/mimir-mixin/alerts/compactor.libsonnet
@@ -38,10 +38,10 @@
           alert: 'CortexCompactorHasNotUploadedBlocks',
           'for': '15m',
           expr: |||
-            (time() - thanos_objstore_bucket_last_successful_upload_time{job=~".+/compactor"} > 60 * 60 * 24)
+            (time() - thanos_objstore_bucket_last_successful_upload_time{job=~".+/%(compactor)s"} > 60 * 60 * 24)
             and
-            (thanos_objstore_bucket_last_successful_upload_time{job=~".+/compactor"} > 0)
-          |||,
+            (thanos_objstore_bucket_last_successful_upload_time{job=~".+/%(compactor)s"} > 0)
+          ||| % $._config.job_names,
           labels: {
             severity: 'critical',
           },
@@ -54,8 +54,8 @@
           alert: 'CortexCompactorHasNotUploadedBlocksSinceStart',
           'for': '24h',
           expr: |||
-            thanos_objstore_bucket_last_successful_upload_time{job=~".+/compactor"} == 0
-          |||,
+            thanos_objstore_bucket_last_successful_upload_time{job=~".+/%(compactor)s"} == 0
+          ||| % $._config.job_names,
           labels: {
             severity: 'critical',
           },

diff --git a/jsonnet/mimir-mixin/config.libsonnet b/jsonnet/mimir-mixin/config.libsonnet
@@ -35,6 +35,7 @@
       table_manager: '(table-manager|cortex$)',
       store_gateway: '(store-gateway|cortex$)',
       gateway: 'cortex-gw',
+      compactor: 'compactor.*',  // Match also custom compactor deployments.
     },
 
     // Labels used to in alert aggregations - should uniquely identify

diff --git a/jsonnet/mimir-mixin/dashboards/compactor-resources.libsonnet b/jsonnet/mimir-mixin/dashboards/compactor-resources.libsonnet
@@ -13,7 +13,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.containerMemoryWorkingSetPanel('Memory (workingset)', 'compactor'),
       )
       .addPanel(
-        $.goHeapInUsePanel('Memory (go heap inuse)', 'compactor'),
+        $.goHeapInUsePanel('Memory (go heap inuse)', $._config.job_names.compactor),
       )
     )
     .addRow(

diff --git a/jsonnet/mimir-mixin/dashboards/compactor.libsonnet b/jsonnet/mimir-mixin/dashboards/compactor.libsonnet
@@ -15,9 +15,9 @@ local utils = import 'mixin-utils/utils.libsonnet';
       .addPanel(
         $.startedCompletedFailedPanel(
           'Per-instance runs / sec',
-          'sum(rate(cortex_compactor_runs_started_total{%s}[$__rate_interval]))' % $.jobMatcher('compactor'),
-          'sum(rate(cortex_compactor_runs_completed_total{%s}[$__rate_interval]))' % $.jobMatcher('compactor'),
-          'sum(rate(cortex_compactor_runs_failed_total{%s}[$__rate_interval]))' % $.jobMatcher('compactor')
+          'sum(rate(cortex_compactor_runs_started_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.compactor),
+          'sum(rate(cortex_compactor_runs_completed_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.compactor),
+          'sum(rate(cortex_compactor_runs_failed_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.compactor)
         ) +
         $.bars +
         { yaxes: $.yaxes('ops') },
@@ -30,7 +30,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
             cortex_compactor_tenants_processing_failed{%s} +
             cortex_compactor_tenants_skipped{%s}
           ) / cortex_compactor_tenants_discovered{%s}
-        ||| % [$.jobMatcher('compactor'), $.jobMatcher('compactor'), $.jobMatcher('compactor'), $.jobMatcher('compactor')], '{{%s}}' % $._config.per_instance_label) +
+        ||| % [$.jobMatcher($._config.job_names.compactor), $.jobMatcher($._config.job_names.compactor), $.jobMatcher($._config.job_names.compactor), $.jobMatcher($._config.job_names.compactor)], '{{%s}}' % $._config.per_instance_label) +
         { yaxes: $.yaxes({ format: 'percentunit', max: 1 }) },
       )
     )
@@ -44,12 +44,12 @@ local utils = import 'mixin-utils/utils.libsonnet';
       )
       .addPanel(
         $.panel('Compacted blocks / sec') +
-        $.queryPanel('sum(rate(prometheus_tsdb_compactions_total{%s}[$__rate_interval]))' % $.jobMatcher('compactor'), 'blocks') +
+        $.queryPanel('sum(rate(prometheus_tsdb_compactions_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.compactor), 'blocks') +
         { yaxes: $.yaxes('ops') },
       )
       .addPanel(
         $.panel('Per-block compaction duration') +
-        $.latencyPanel('prometheus_tsdb_compaction_duration_seconds', '{%s}' % $.jobMatcher('compactor'))
+        $.latencyPanel('prometheus_tsdb_compaction_duration_seconds', '{%s}' % $.jobMatcher($._config.job_names.compactor))
       )
     )
     .addRow(
@@ -62,27 +62,27 @@ local utils = import 'mixin-utils/utils.libsonnet';
       )
       .addPanel(
         $.panel('Average blocks / tenant') +
-        $.queryPanel('avg(max by(user) (cortex_bucket_blocks_count{%s}))' % $.jobMatcher('compactor'), 'avg'),
+        $.queryPanel('avg(max by(user) (cortex_bucket_blocks_count{%s}))' % $.jobMatcher($._config.job_names.compactor), 'avg'),
       )
       .addPanel(
         $.panel('Tenants with largest number of blocks') +
-        $.queryPanel('topk(10, max by(user) (cortex_bucket_blocks_count{%s}))' % $.jobMatcher('compactor'), '{{user}}'),
+        $.queryPanel('topk(10, max by(user) (cortex_bucket_blocks_count{%s}))' % $.jobMatcher($._config.job_names.compactor), '{{user}}'),
       )
     )
     .addRow(
       $.row('Garbage Collector')
       .addPanel(
         $.panel('Blocks marked for deletion / sec') +
-        $.queryPanel('sum(rate(cortex_compactor_blocks_marked_for_deletion_total{%s}[$__rate_interval]))' % $.jobMatcher('compactor'), 'blocks') +
+        $.queryPanel('sum(rate(cortex_compactor_blocks_marked_for_deletion_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.compactor), 'blocks') +
         { yaxes: $.yaxes('ops') },
       )
       .addPanel(
         $.successFailurePanel(
           'Blocks deletions / sec',
           // The cortex_compactor_blocks_cleaned_total tracks the number of successfully
           // deleted blocks.
-          'sum(rate(cortex_compactor_blocks_cleaned_total{%s}[$__rate_interval]))' % $.jobMatcher('compactor'),
-          'sum(rate(cortex_compactor_block_cleanup_failures_total{%s}[$__rate_interval]))' % $.jobMatcher('compactor'),
+          'sum(rate(cortex_compactor_blocks_cleaned_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.compactor),
+          'sum(rate(cortex_compactor_block_cleanup_failures_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.compactor),
         ) + { yaxes: $.yaxes('ops') }
       )
     )
@@ -93,14 +93,14 @@ local utils = import 'mixin-utils/utils.libsonnet';
           'Metadata Syncs / sec',
           // The cortex_compactor_meta_syncs_total metric is incremented each time a per-tenant
           // metadata sync is triggered.
-          'sum(rate(cortex_compactor_meta_syncs_total{%s}[$__rate_interval])) - sum(rate(cortex_compactor_meta_sync_failures_total{%s}[$__rate_interval]))' % [$.jobMatcher('compactor'), $.jobMatcher('compactor')],
-          'sum(rate(cortex_compactor_meta_sync_failures_total{%s}[$__rate_interval]))' % $.jobMatcher('compactor'),
+          'sum(rate(cortex_compactor_meta_syncs_total{%s}[$__rate_interval])) - sum(rate(cortex_compactor_meta_sync_failures_total{%s}[$__rate_interval]))' % [$.jobMatcher($._config.job_names.compactor), $.jobMatcher($._config.job_names.compactor)],
+          'sum(rate(cortex_compactor_meta_sync_failures_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.compactor),
         ) + { yaxes: $.yaxes('ops') }
       )
       .addPanel(
         $.panel('Metadata Sync Duration') +
         // This metric tracks the duration of a per-tenant metadata sync.
-        $.latencyPanel('cortex_compactor_meta_sync_duration_seconds', '{%s}' % $.jobMatcher('compactor')),
+        $.latencyPanel('cortex_compactor_meta_sync_duration_seconds', '{%s}' % $.jobMatcher($._config.job_names.compactor)),
       )
     )
     .addRow($.objectStorePanels1('Object Store', 'compactor'))