Alloy-Mixin: allow config overwrite (#934)

* Allow config overwrite, and make grafanaDashboards and prometheusAlerts hidden * other bugfixes * applying filterselector to metrics and logs * fix logs dashboard tag * remove whitespaces * allowing empty filterSelectors and disabling SentenceCase labels * allow useSentenceCaseLabel * rename useSetenceCaseTemplateLabels config option * fixing sentencecase config param
grafana · Jun 6, 2024 · 10da36a · 10da36a
1 parent 0b9862d
commit 10da36a
Show file tree

Hide file tree

Showing 13 changed files with 287 additions and 104 deletions.
diff --git a/example/config/alloy/config.alloy b/example/config/alloy/config.alloy
@@ -8,7 +8,21 @@ logging {
 	level = "debug"
 
 	// Forward internal logs to the local Loki instance.
-	write_to = [loki.write.loki.receiver]
+	write_to = [loki.relabel.alloy_logs.receiver]
+}
+
+loki.relabel "alloy_logs"{
+	rule {
+		target_label = "instance"
+		replacement = constants.hostname
+	}
+
+	rule {
+		target_label = "job"
+		replacement = "integrations/self"
+	}
+
+	forward_to = [loki.write.loki.receiver]
 }
 
 tracing {

diff --git a/operations/alloy-mixin/alerts.libsonnet b/operations/alloy-mixin/alerts.libsonnet
@@ -10,7 +10,7 @@ local openTelemetryAlerts = (import './alerts/opentelemetry.libsonnet');
     openTelemetryAlerts.newOpenTelemetryAlertsGroup($._config.enableK8sCluster)
   ],
 
-  prometheusAlerts+: {
+  prometheusAlerts+:: {
     groups+: 
       if $._config.enableAlloyCluster then
         alloyClusterAlerts + otherAlerts

diff --git a/operations/alloy-mixin/config.libsonnet b/operations/alloy-mixin/config.libsonnet
@@ -3,10 +3,12 @@
         enableK8sCluster: true,
         enableAlloyCluster: true,
         enableLokiLogs: true,
-        filterSelector: 'job=~"$job"',
-        groupSelector: if self.enableK8sCluster then self.k8sClusterSelector + ', ' + self.filterSelector else self.filterSelector,
-        instanceSelector: self.groupSelector + ', instance=~"$instance"',
+        filterSelector: '', #use it to filter specific metric label values, ie: job=~"integrations/alloy"
         k8sClusterSelector: 'cluster=~"$cluster", namespace=~"$namespace"',
-        dashboardTag: 'alloy-mixin'
+        groupSelector: if self.enableK8sCluster then self.k8sClusterSelector + ', job="$job"' else 'job="$job"',        
+        instanceSelector: self.groupSelector + ', instance=~"$instance"',        
+        logsFilterSelector: '', #use to filter logs originated from alloy, and avoid picking up other platform logs, ie: service_name="alloy"
+        dashboardTag: 'alloy-mixin',
+        useSetenceCaseTemplateLabels: false,
     }
 }
diff --git a/operations/alloy-mixin/dashboards.libsonnet b/operations/alloy-mixin/dashboards.libsonnet
@@ -1,21 +1,21 @@
-local alloyClusterDashboards =   
-  (import './dashboards/cluster-node.libsonnet') + 
-  (import './dashboards/cluster-overview.libsonnet') +
-  (import './config.libsonnet');
-
-local otherDashboards =  
-  (import './dashboards/resources.libsonnet') +
-  (import './dashboards/controller.libsonnet') + 
-  (import './dashboards/prometheus.libsonnet') + 
-  (import './dashboards/opentelemetry.libsonnet') +
-  (import './config.libsonnet');
-
 (import './dashboards/alloy-logs.libsonnet') +
-{   
-  grafanaDashboards+:     
+{
+  local alloyClusterDashboards =   
+    (import './dashboards/cluster-node.libsonnet') + 
+    (import './dashboards/cluster-overview.libsonnet'),
+  local otherDashboards =  
+    (import './dashboards/resources.libsonnet') +
+    (import './dashboards/controller.libsonnet') + 
+    (import './dashboards/prometheus.libsonnet') + 
+    (import './dashboards/opentelemetry.libsonnet'),
+
+  grafanaDashboards+::
+  // Propagate config down to inner dashboards.
+  { _config:: $._config } + (
     if $._config.enableAlloyCluster then 
-       alloyClusterDashboards +
-       otherDashboards
-    else
+      alloyClusterDashboards + 
       otherDashboards
+    else 
+      otherDashboards 
+  )
 }
diff --git a/operations/alloy-mixin/dashboards/alloy-logs.libsonnet b/operations/alloy-mixin/dashboards/alloy-logs.libsonnet
@@ -9,10 +9,10 @@ local logsDashboard = import 'github.com/grafana/jsonnet-libs/logs-lib/logs/main
     if $._config.enableLokiLogs then {
       local alloyLogs =
         logsDashboard.new(
-          'Alloy logs overview',
+          'Alloy / Logs Overview',
           datasourceName='loki_datasource',
           datasourceRegex='',
-          filterSelector=$._config.filterSelector,
+          filterSelector=$._config.logsFilterSelector,
           labels=labels,
           formatParser=null,
           showLogsVolume=true
@@ -27,7 +27,8 @@ local logsDashboard = import 'github.com/grafana/jsonnet-libs/logs-lib/logs/main
           dashboards+:
             {
               logs+: g.dashboard.withLinksMixin($.grafanaDashboards['alloy-resources.json'].links)                     
-                     + g.dashboard.withRefresh('10s'),
+                     + g.dashboard.withRefresh('10s')
+                     + g.dashboard.withTagsMixin($._config.dashboardTag),
             },
         },
       'alloy-logs.json': alloyLogs.dashboards.logs,

diff --git a/operations/alloy-mixin/dashboards/cluster-node.libsonnet b/operations/alloy-mixin/dashboards/cluster-node.libsonnet
@@ -1,21 +1,15 @@
 local dashboard = import './utils/dashboard.jsonnet';
 local panel = import './utils/panel.jsonnet';
+local templates = import './utils/templates.libsonnet';
 local filename = 'alloy-cluster-node.json';
 
 {
   local templateVariables = 
-    if $._config.enableK8sCluster then
-      [
-        dashboard.newTemplateVariable('cluster', 'label_values(alloy_component_controller_running_components, cluster)'),
-        dashboard.newTemplateVariable('namespace', 'label_values(alloy_component_controller_running_components{cluster=~"$cluster"}, namespace)'),
-        dashboard.newMultiTemplateVariable('job', 'label_values(alloy_component_controller_running_components{cluster=~"$cluster", namespace=~"$namespace"}, job)'),
-        dashboard.newMultiTemplateVariable('instance', 'label_values(alloy_component_controller_running_components{cluster=~"$cluster", namespace=~"$namespace", job=~"$job"}, instance)'),
-      ]
-    else
-      [
-        dashboard.newMultiTemplateVariable('job', 'label_values(alloy_component_controller_running_components, job)'),
-        dashboard.newMultiTemplateVariable('instance', 'label_values(alloy_component_controller_running_components{job=~"$job"}, instance)'),
-      ],
+    templates.newTemplateVariablesList(
+      filterSelector=$._config.filterSelector, 
+      enableK8sCluster=$._config.enableK8sCluster, 
+      includeInstance=true,
+      setenceCaseLabels=$._config.useSetenceCaseTemplateLabels),
 
   [filename]:
     dashboard.new(name='Alloy / Cluster Node', tag=$._config.dashboardTag) +

diff --git a/operations/alloy-mixin/dashboards/cluster-overview.libsonnet b/operations/alloy-mixin/dashboards/cluster-overview.libsonnet
@@ -1,20 +1,16 @@
 local dashboard = import './utils/dashboard.jsonnet';
 local panel = import './utils/panel.jsonnet';
 local filename = 'alloy-cluster-overview.json';
+local templates = import './utils/templates.libsonnet';
 local cluster_node_filename = 'alloy-cluster-node.json';
 
 {
   local templateVariables = 
-    if $._config.enableK8sCluster then
-      [
-        dashboard.newTemplateVariable('cluster', 'label_values(alloy_component_controller_running_components, cluster)'),
-        dashboard.newTemplateVariable('namespace', 'label_values(alloy_component_controller_running_components{cluster=~"$cluster"}, namespace)'),
-        dashboard.newMultiTemplateVariable('job', 'label_values(alloy_component_controller_running_components{cluster=~"$cluster", namespace=~"$namespace"}, job)'),
-      ]
-    else
-      [
-        dashboard.newMultiTemplateVariable('job', 'label_values(alloy_component_controller_running_components, job)'),        
-      ],
+    templates.newTemplateVariablesList(
+      filterSelector=$._config.filterSelector, 
+      enableK8sCluster=$._config.enableK8sCluster, 
+      includeInstance=false,
+      setenceCaseLabels=$._config.useSetenceCaseTemplateLabels),
 
   [filename]:
     dashboard.new(name='Alloy / Cluster Overview', tag=$._config.dashboardTag) +
@@ -36,7 +32,9 @@ local cluster_node_filename = 'alloy-cluster-node.json';
         panel.withPosition({ h: 9, w: 8, x: 0, y: 0 }) +
         panel.withQueries([
           panel.newInstantQuery(
-            expr='count(cluster_node_info{%(groupSelector)s})'
+            expr= |||
+              'count(cluster_node_info{%(groupSelector)s})'
+            ||| % $._config
           ),
         ])
       ),
@@ -49,7 +47,9 @@ local cluster_node_filename = 'alloy-cluster-node.json';
         panel.withPosition({ h: 9, w: 16, x: 8, y: 0 }) +
         panel.withQueries([
           panel.newInstantQuery(
-            expr='cluster_node_info{%(groupSelector)s}',
+            expr= |||
+              'cluster_node_info{%(groupSelector)s}'
+            ||| % $._config,
             format='table',
           ),
         ]) +

diff --git a/operations/alloy-mixin/dashboards/controller.libsonnet b/operations/alloy-mixin/dashboards/controller.libsonnet
@@ -1,20 +1,15 @@
 local dashboard = import './utils/dashboard.jsonnet';
 local panel = import './utils/panel.jsonnet';
+local templates = import './utils/templates.libsonnet';
 local filename = 'alloy-controller.json';
 
 {
-
   local templateVariables = 
-    if $._config.enableK8sCluster then
-      [
-        dashboard.newTemplateVariable('cluster', 'label_values(alloy_component_controller_running_components, cluster)'),
-        dashboard.newTemplateVariable('namespace', 'label_values(alloy_component_controller_running_components{cluster=~"$cluster"}, namespace)'),
-        dashboard.newMultiTemplateVariable('job', 'label_values(alloy_component_controller_running_components{cluster=~"$cluster", namespace=~"$namespace"}, job)'),
-      ]
-    else
-      [
-        dashboard.newMultiTemplateVariable('job', 'label_values(alloy_component_controller_running_components, job)'),        
-      ],
+    templates.newTemplateVariablesList(
+      filterSelector=$._config.filterSelector, 
+      enableK8sCluster=$._config.enableK8sCluster, 
+      includeInstance=false,
+      setenceCaseLabels=$._config.useSetenceCaseTemplateLabels),
 
   [filename]:
     dashboard.new(name='Alloy / Controller', tag=$._config.dashboardTag) +
@@ -309,7 +304,7 @@ local filename = 'alloy-controller.json';
             expr= |||
               sum(increase(alloy_component_evaluation_seconds{%(groupSelector)s}[$__rate_interval]))
               or ignoring (le)
-              sum by (le) (increase(alloy_component_evaluation_seconds_bucket{%(groupSelector)s}[$__rate_interval]))'
+              sum by (le) (increase(alloy_component_evaluation_seconds_bucket{%(groupSelector)s}[$__rate_interval]))
             ||| % $._config,
             format='heatmap',
             legendFormat='{{le}}',

diff --git a/operations/alloy-mixin/dashboards/opentelemetry.libsonnet b/operations/alloy-mixin/dashboards/opentelemetry.libsonnet
@@ -1,5 +1,6 @@
 local dashboard = import './utils/dashboard.jsonnet';
 local panel = import './utils/panel.jsonnet';
+local templates = import './utils/templates.libsonnet';
 local filename = 'alloy-opentelemetry.json';
 
 local stackedPanelMixin = {
@@ -16,18 +17,11 @@ local stackedPanelMixin = {
 
 {
   local templateVariables = 
-    if $._config.enableK8sCluster then
-      [
-        dashboard.newTemplateVariable('cluster', 'label_values(alloy_component_controller_running_components, cluster)'),
-        dashboard.newTemplateVariable('namespace', 'label_values(alloy_component_controller_running_components{cluster=~"$cluster"}, namespace)'),
-        dashboard.newMultiTemplateVariable('job', 'label_values(alloy_component_controller_running_components{cluster=~"$cluster", namespace=~"$namespace"}, job)'),
-        dashboard.newMultiTemplateVariable('instance', 'label_values(alloy_component_controller_running_components{cluster=~"$cluster", namespace=~"$namespace", job=~"$job"}, instance)'),
-      ]
-    else
-      [
-        dashboard.newMultiTemplateVariable('job', 'label_values(alloy_component_controller_running_components, job)'),
-        dashboard.newMultiTemplateVariable('instance', 'label_values(alloy_component_controller_running_components{job=~"$job"}, instance)'),
-      ],
+    templates.newTemplateVariablesList(
+      filterSelector=$._config.filterSelector, 
+      enableK8sCluster=$._config.enableK8sCluster, 
+      includeInstance=true,
+      setenceCaseLabels=$._config.useSetenceCaseTemplateLabels),
 
   [filename]:
     dashboard.new(name='Alloy / OpenTelemetry', tag=$._config.dashboardTag) +

diff --git a/operations/alloy-mixin/dashboards/prometheus.libsonnet b/operations/alloy-mixin/dashboards/prometheus.libsonnet
@@ -1,5 +1,6 @@
 local dashboard = import './utils/dashboard.jsonnet';
 local panel = import './utils/panel.jsonnet';
+local templates = import './utils/templates.libsonnet';
 local filename = 'alloy-prometheus-remote-write.json';
 
 {
@@ -397,25 +398,105 @@ local filename = 'alloy-prometheus-remote-write.json';
     else
       remoteWritePanels(y_offset=0),
 
-  local templateVariables = 
+  local k8sComponentPathQuery = 
+    if std.isEmpty($._config.filterSelector) then
+    |||
+        label_values(prometheus_remote_write_wal_samples_appended_total{cluster=~"$cluster", namespace=~"$namespace", job=~"$job", instance=~"$instance", component_id=~"prometheus.remote_write.*", component_path=~".*"}, component_path)
+    |||
+    else
+    |||
+        label_values(prometheus_remote_write_wal_samples_appended_total{%(filterSelector)s, cluster=~"$cluster", namespace=~"$namespace", job=~"$job", instance=~"$instance", component_id=~"prometheus.remote_write.*", component_path=~".*"}, component_path)
+    ||| % $._config,
+
+  local k8sComponentQuery = 
+    if std.isEmpty($._config.filterSelector) then
+    |||
+        label_values(prometheus_remote_write_wal_samples_appended_total{cluster=~"$cluster", namespace=~"$namespace", job=~"$job", instance=~"$instance", component_id=~"prometheus.remote_write.*"}, component_id)
+    |||
+    else
+    |||
+        label_values(prometheus_remote_write_wal_samples_appended_total{%(filterSelector)s, cluster=~"$cluster", namespace=~"$namespace", job=~"$job", instance=~"$instance", component_id=~"prometheus.remote_write.*"}, component_id)
+    ||| % $._config,
+
+  local k8sUrlQuery = 
+    if std.isEmpty($._config.filterSelector) then
+    |||
+        label_values(prometheus_remote_storage_sent_batch_duration_seconds_sum{cluster=~"$cluster", namespace=~"$namespace", job="$job", instance=~"$instance", component_id=~"$component"}, url)
+    |||
+    else
+    |||
+        label_values(prometheus_remote_storage_sent_batch_duration_seconds_sum{%(filterSelector)s, cluster=~"$cluster", namespace=~"$namespace", job="$job", instance=~"$instance", component_id=~"$component"}, url)
+    ||| % $._config,
+
+  local componentPathQuery = 
+    if std.isEmpty($._config.filterSelector) then
+    |||
+        label_values(prometheus_remote_write_wal_samples_appended_total{job=~"$job", instance=~"$instance", component_id=~"prometheus.remote_write.*", component_path=~".*"}, component_path)
+    |||
+    else
+    |||
+        label_values(prometheus_remote_write_wal_samples_appended_total{%(filterSelector)s, job=~"$job", instance=~"$instance", component_id=~"prometheus.remote_write.*", component_path=~".*"}, component_path)
+    ||| % $._config,
+
+  local componentQuery = 
+    if std.isEmpty($._config.filterSelector) then
+    |||
+        label_values(prometheus_remote_write_wal_samples_appended_total{job=~"$job", instance=~"$instance", component_id=~"prometheus.remote_write.*"}, component_id)
+    |||
+    else
+    |||
+        label_values(prometheus_remote_write_wal_samples_appended_total{%(filterSelector)s, job=~"$job", instance=~"$instance", component_id=~"prometheus.remote_write.*"}, component_id)
+    ||| % $._config,
+
+  local urlQuery = 
+    if std.isEmpty($._config.filterSelector) then
+    |||
+        label_values(prometheus_remote_storage_sent_batch_duration_seconds_sum{job="$job", instance=~"$instance", component_id=~"$component"}, url)
+    |||
+    else
+    |||
+        label_values(prometheus_remote_storage_sent_batch_duration_seconds_sum{%(filterSelector)s, job="$job", instance=~"$instance", component_id=~"$component"}, url)
+    ||| % $._config,
+
+  local prometheusTemplateVariables =
     if $._config.enableK8sCluster then
-      [
-        dashboard.newTemplateVariable('cluster', 'label_values(alloy_component_controller_running_components, cluster)'),
-        dashboard.newTemplateVariable('namespace', 'label_values(alloy_component_controller_running_components{cluster=~"$cluster"}, namespace)'),
-        dashboard.newMultiTemplateVariable('job', 'label_values(alloy_component_controller_running_components{cluster=~"$cluster", namespace=~"$namespace"}, job)'),
-        dashboard.newMultiTemplateVariable('instance', 'label_values(alloy_component_controller_running_components{cluster=~"$cluster", namespace=~"$namespace", job=~"$job"}, instance)'),
-        dashboard.newMultiTemplateVariable('component_path', 'label_values(prometheus_remote_write_wal_samples_appended_total{cluster=~"$cluster", namespace=~"$namespace", job=~"$job", instance=~"$instance", component_id=~"prometheus.remote_write.*", component_path=~".*"}, component_path)'),
-        dashboard.newMultiTemplateVariable('component', 'label_values(prometheus_remote_write_wal_samples_appended_total{cluster=~"$cluster", namespace=~"$namespace", job=~"$job", instance=~"$instance", component_id=~"prometheus.remote_write.*"}, component_id)'),
-        dashboard.newMultiTemplateVariable('url', 'label_values(prometheus_remote_storage_sent_batch_duration_seconds_sum{cluster=~"$cluster", namespace=~"$namespace", job="$job", instance=~"$instance", component_id=~"$component"}, url)'),
+      [        
+        dashboard.newMultiTemplateVariable(
+          name='component_path', 
+          query=k8sComponentPathQuery,
+          setenceCaseLabels=$._config.useSetenceCaseTemplateLabels),
+        dashboard.newMultiTemplateVariable(
+          name='component', 
+          query=k8sComponentQuery,
+          setenceCaseLabels=$._config.useSetenceCaseTemplateLabels),          
+        dashboard.newMultiTemplateVariable(
+          name='url', 
+          query= k8sUrlQuery,
+          setenceCaseLabels=$._config.useSetenceCaseTemplateLabels),
       ]
     else
-      [
-        dashboard.newMultiTemplateVariable('job', 'label_values(alloy_component_controller_running_components, job)'),
-        dashboard.newMultiTemplateVariable('instance', 'label_values(alloy_component_controller_running_components{job=~"$job"}, instance)'),
-        dashboard.newMultiTemplateVariable('component_path', 'label_values(prometheus_remote_write_wal_samples_appended_total{job=~"$job", instance=~"$instance", component_id=~"prometheus.remote_write.*", component_path=~".*"}, component_path)'),
-        dashboard.newMultiTemplateVariable('component', 'label_values(prometheus_remote_write_wal_samples_appended_total{job=~"$job", instance=~"$instance", component_id=~"prometheus.remote_write.*"}, component_id)'),
-        dashboard.newMultiTemplateVariable('url', 'label_values(prometheus_remote_storage_sent_batch_duration_seconds_sum{job=~"$job", instance=~"$instance", component_id=~"$component"}, url)'),
+      [       
+        dashboard.newMultiTemplateVariable(
+          name='component_path', 
+          query=componentPathQuery,
+          setenceCaseLabels=$._config.useSetenceCaseTemplateLabels),
+        dashboard.newMultiTemplateVariable(
+          name='component', 
+          query=componentQuery,
+          setenceCaseLabels=$._config.useSetenceCaseTemplateLabels),          
+        dashboard.newMultiTemplateVariable(
+          name='url', 
+          query=urlQuery,
+          setenceCaseLabels=$._config.useSetenceCaseTemplateLabels),
       ],
+
+    local templateVariables = 
+      templates.newTemplateVariablesList(
+        filterSelector=$._config.filterSelector, 
+        enableK8sCluster=$._config.enableK8sCluster, 
+        includeInstance=true,
+        setenceCaseLabels=$._config.useSetenceCaseTemplateLabels)
+      + prometheusTemplateVariables,
 
   [filename]:
     dashboard.new(name='Alloy / Prometheus Components', tag=$._config.dashboardTag) +