feature: Add text boxes and descriptions to reads and writes dashboards

CHANGELOG.md

@@ -17,6 +17,7 @@
 * [CHANGE] Renamed `CortexInconsistentConfig` alert to `CortexInconsistentRuntimeConfig` and increased severity to `critical`. #335
 * [CHANGE] Increased `CortexBadRuntimeConfig` alert severity to `critical` and removed support for `cortex_overrides_last_reload_successful` metric (was removed in Cortex 1.3.0). #335
 * [ENHANCEMENT] cortex-mixin: Make `cluster_namespace_deployment:kube_pod_container_resource_requests_{cpu_cores,memory_bytes}:sum` backwards compatible with `kube-state-metrics` v2.0.0. #317
+* [ENHANCEMENT] Added documentation text panels and descriptions to reads and writes dashboards. #324
 * [BUGFIX] Fixed `CortexIngesterHasNotShippedBlocks` alert false positive in case an ingester instance had ingested samples in the past, then no traffic was received for a long period and then it started receiving samples again. #308
 * [BUGFIX] Alertmanager: fixed `--alertmanager.cluster.peers` CLI flag passed to alertmanager when HA is enabled. #329
 * [BUGFIX] Fixed `CortexInconsistentRuntimeConfig` metric. #335

cortex-mixin/config.libsonnet

@@ -58,5 +58,11 @@
 
     // The label used to differentiate between different nodes (i.e. servers).
     per_node_label: 'instance',
+
+    // Whether certain dashboard description headers should be shown
+    show_dashboard_descriptions: {
+      writes: true,
+      reads: true,
+    },
   },
 }

cortex-mixin/dashboards/compactor.libsonnet

@@ -6,12 +6,6 @@ local utils = import 'mixin-utils/utils.libsonnet';
     .addClusterSelectorTemplates()
     .addRow(
       $.row('Summary')
-      .addPanel(
-        $.textPanel('', |||
-          - **Per-instance runs**: number of times a compactor instance triggers a compaction across all tenants its shard manage.
-          - **Tenants compaction progress**: in a multi-tenant cluster it shows the progress of tenants compacted while compaction is running. Reset to 0 once the compaction run is completed for all tenants in the shard.
-        |||),
-      )
       .addPanel(
         $.startedCompletedFailedPanel(
           'Per-instance runs / sec',
@@ -20,7 +14,13 @@ local utils = import 'mixin-utils/utils.libsonnet';
           'sum(rate(cortex_compactor_runs_failed_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.compactor)
         ) +
         $.bars +
-        { yaxes: $.yaxes('ops') },
+        { yaxes: $.yaxes('ops') } +
+        $.panelDescription(
+          'Per-instance runs',
+          |||
+            Number of times a compactor instance triggers a compaction across all tenants that it manages.
+          |||
+        ),
       )
       .addPanel(
         $.panel('Tenants compaction progress') +
@@ -31,42 +31,55 @@ local utils = import 'mixin-utils/utils.libsonnet';
             cortex_compactor_tenants_skipped{%s}
           ) / cortex_compactor_tenants_discovered{%s}
         ||| % [$.jobMatcher($._config.job_names.compactor), $.jobMatcher($._config.job_names.compactor), $.jobMatcher($._config.job_names.compactor), $.jobMatcher($._config.job_names.compactor)], '{{%s}}' % $._config.per_instance_label) +
-        { yaxes: $.yaxes({ format: 'percentunit', max: 1 }) },
+        { yaxes: $.yaxes({ format: 'percentunit', max: 1 }) } +
+        $.panelDescription(
+          'Tenants compaction progress',
+          |||
+            In a multi-tenant cluster, display the progress of tenants that are compacted while compaction is running.
+            Reset to <tt>0</tt> after the compaction run is completed for all tenants in the shard.
+          |||
+        ),
       )
     )
     .addRow(
       $.row('')
-      .addPanel(
-        $.textPanel('', |||
-          - **Compacted blocks**: number of blocks generated as a result of a compaction operation.
-          - **Per-block compaction duration**: time taken to generate a single compacted block.
-        |||),
-      )
       .addPanel(
         $.panel('Compacted blocks / sec') +
         $.queryPanel('sum(rate(prometheus_tsdb_compactions_total{%s}[$__rate_interval]))' % $.jobMatcher($._config.job_names.compactor), 'blocks') +
-        { yaxes: $.yaxes('ops') },
+        { yaxes: $.yaxes('ops') } +
+        $.panelDescription(
+          'Compacted blocks / sec',
+          |||
+            Rate of blocks that are generated as a result of a compaction operation.
+          |||
+        ),
       )
       .addPanel(
         $.panel('Per-block compaction duration') +
-        $.latencyPanel('prometheus_tsdb_compaction_duration_seconds', '{%s}' % $.jobMatcher($._config.job_names.compactor))
+        $.latencyPanel('prometheus_tsdb_compaction_duration_seconds', '{%s}' % $.jobMatcher($._config.job_names.compactor)) +
+        $.panelDescription(
+          'Per-block compaction duration',
+          |||
+            Display the amount of time that it has taken to generate a single compacted block.
+          |||
+        ),
       )
     )
     .addRow(
       $.row('')
-      .addPanel(
-        $.textPanel('', |||
-          - **Average blocks / tenant**: the average number of blocks per tenant.
-          - **Tenants with largest number of blocks**: the 10 tenants with the largest number of blocks.
-        |||),
-      )
       .addPanel(
         $.panel('Average blocks / tenant') +
         $.queryPanel('avg(max by(user) (cortex_bucket_blocks_count{%s}))' % $.jobMatcher($._config.job_names.compactor), 'avg'),
       )
       .addPanel(
         $.panel('Tenants with largest number of blocks') +
-        $.queryPanel('topk(10, max by(user) (cortex_bucket_blocks_count{%s}))' % $.jobMatcher($._config.job_names.compactor), '{{user}}'),
+        $.queryPanel('topk(10, max by(user) (cortex_bucket_blocks_count{%s}))' % $.jobMatcher($._config.job_names.compactor), '{{user}}') +
+        $.panelDescription(
+          'Tenants with largest number of blocks',
+          |||
+            The 10 tenants with the largest number of blocks.
+          |||
+        ),
       )
     )
     .addRow(
@@ -103,6 +116,5 @@ local utils = import 'mixin-utils/utils.libsonnet';
         $.latencyPanel('cortex_compactor_meta_sync_duration_seconds', '{%s}' % $.jobMatcher($._config.job_names.compactor)),
       )
     )
-    .addRow($.objectStorePanels1('Object Store', 'compactor'))
-    .addRow($.objectStorePanels2('', 'compactor')),
+    .addRows($.getObjectStoreRows('Object Store', 'compactor')),
 }

cortex-mixin/dashboards/dashboard-utils.libsonnet

@@ -14,6 +14,24 @@ local utils = import 'mixin-utils/utils.libsonnet';
         then self.addRow(row)
         else self,
 
+      addRowsIf(condition, rows)::
+        if condition
+        then
+          local reduceRows(dashboard, remainingRows) =
+            if (std.length(remainingRows) == 0)
+            then dashboard
+            else
+              reduceRows(
+                dashboard.addRow(remainingRows[0]),
+                std.slice(remainingRows, 1, std.length(remainingRows), 1)
+              )
+          ;
+          reduceRows(self, rows)
+        else self,
+
+      addRows(rows)::
+        self.addRowsIf(true, rows),
+
       addClusterSelectorTemplates(multi=true)::
         local d = self {
           tags: $._config.tags,
@@ -43,7 +61,6 @@ local utils = import 'mixin-utils/utils.libsonnet';
           else d
                .addTemplate('cluster', 'cortex_build_info', 'cluster')
                .addTemplate('namespace', 'cortex_build_info{cluster=~"$cluster"}', 'namespace'),
-
     },
 
   // The mixin allow specialism of the job selector depending on if its a single binary
@@ -274,7 +291,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
     type: 'text',
   } + options,
 
-  objectStorePanels1(title, component)::
+  getObjectStoreRows(title, component):: [
     super.row(title)
     .addPanel(
       $.panel('Operations / sec') +
@@ -288,62 +305,135 @@ local utils = import 'mixin-utils/utils.libsonnet';
       { yaxes: $.yaxes('percentunit') },
     )
     .addPanel(
-      $.panel('Op: Attributes') +
+      $.panel('Latency of Op: Attributes') +
       $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="%s",operation="attributes"}' % [$.namespaceMatcher(), component]),
     )
     .addPanel(
-      $.panel('Op: Exists') +
+      $.panel('Latency of Op: Exists') +
       $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="%s",operation="exists"}' % [$.namespaceMatcher(), component]),
     ),
-
-  // Second row of Object Store stats
-  objectStorePanels2(title, component)::
-    super.row(title)
+    $.row('')
     .addPanel(
-      $.panel('Op: Get') +
+      $.panel('Latency of Op: Get') +
       $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="%s",operation="get"}' % [$.namespaceMatcher(), component]),
     )
     .addPanel(
-      $.panel('Op: GetRange') +
+      $.panel('Latency of Op: GetRange') +
       $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="%s",operation="get_range"}' % [$.namespaceMatcher(), component]),
     )
     .addPanel(
-      $.panel('Op: Upload') +
+      $.panel('Latency of Op: Upload') +
       $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="%s",operation="upload"}' % [$.namespaceMatcher(), component]),
     )
     .addPanel(
-      $.panel('Op: Delete') +
+      $.panel('Latency of Op: Delete') +
       $.latencyPanel('thanos_objstore_bucket_operation_duration_seconds', '{%s,component="%s",operation="delete"}' % [$.namespaceMatcher(), component]),
     ),
+  ],
 
   thanosMemcachedCache(title, jobName, component, cacheName)::
+    local config = {
+      jobMatcher: $.jobMatcher(jobName),
+      component: component,
+      cacheName: cacheName,
+    };
     super.row(title)
     .addPanel(
-      $.panel('QPS') +
-      $.queryPanel('sum by(operation) (rate(thanos_memcached_operations_total{%s,component="%s",name="%s"}[$__rate_interval]))' % [$.jobMatcher(jobName), component, cacheName], '{{operation}}') +
+      $.panel('Requests / sec') +
+      $.queryPanel(
+        |||
+          sum by(operation) (
+            rate(
+              thanos_memcached_operations_total{
+                %(jobMatcher)s,
+                component="%(component)s",
+                name="%(cacheName)s"
+              }[$__rate_interval]
+            )
+          )
+        ||| % config,
+        '{{operation}}'
+      ) +
       $.stack +
-      { yaxes: $.yaxes('ops') },
+      { yaxes: $.yaxes('ops') }
     )
     .addPanel(
       $.panel('Latency (getmulti)') +
-      $.latencyPanel('thanos_memcached_operation_duration_seconds', '{%s,operation="getmulti",component="%s",name="%s"}' % [$.jobMatcher(jobName), component, cacheName])
+      $.latencyPanel(
+        'thanos_memcached_operation_duration_seconds',
+        |||
+          {
+            %(jobMatcher)s,
+            operation="getmulti",
+            component="%(component)s",
+            name="%(cacheName)s"
+          }
+        ||| % config
+      )
     )
     .addPanel(
       $.panel('Hit ratio') +
-      $.queryPanel('sum(rate(thanos_cache_memcached_hits_total{%s,component="%s",name="%s"}[$__rate_interval])) / sum(rate(thanos_cache_memcached_requests_total{%s,component="%s",name="%s"}[$__rate_interval]))' %
-                   [
-                     $.jobMatcher(jobName),
-                     component,
-                     cacheName,
-                     $.jobMatcher(jobName),
-                     component,
-                     cacheName,
-                   ], 'items') +
-      { yaxes: $.yaxes('percentunit') },
+      $.queryPanel(
+        |||
+          sum(
+            rate(
+              thanos_cache_memcached_hits_total{
+                %(jobMatcher)s,
+                component="%(component)s",
+                name="%(cacheName)s"
+              }[$__rate_interval]
+            )
+          ) 
+          / 
+          sum(
+            rate(
+              thanos_cache_memcached_requests_total{
+                %(jobMatcher)s,
+                component="%(component)s",
+                name="%(cacheName)s"
+              }[$__rate_interval]
+            )
+          )
+        ||| % config,
+        'items'
+      ) +
+      { yaxes: $.yaxes('percentunit') }
     ),
 
   filterNodeDiskContainer(containerName)::
     |||
-      ignoring(%s) group_right() (label_replace(count by(%s, %s, device) (container_fs_writes_bytes_total{%s,container="%s",device!~".*sda.*"}), "device", "$1", "device", "/dev/(.*)") * 0)
-    ||| % [$._config.per_instance_label, $._config.per_node_label, $._config.per_instance_label, $.namespaceMatcher(), containerName],
+      ignoring(%s) group_right() (
+        label_replace(
+          count by(
+            %s, 
+            %s, 
+            device
+          ) 
+          (
+            container_fs_writes_bytes_total{
+              %s,
+              container="%s",
+              device!~".*sda.*"
+            }
+          ), 
+          "device", 
+          "$1", 
+          "device", 
+          "/dev/(.*)"
+        ) * 0
+      )
+    ||| % [
+      $._config.per_instance_label,
+      $._config.per_node_label,
+      $._config.per_instance_label,
+      $.namespaceMatcher(),
+      containerName,
+    ],
+
+  panelDescription(title, description):: {
+    description: |||
+      ### %s
+      %s
+    ||| % [title, description],
+  },
 }