Merge pull request #1128 from input-output-hk/jpraynaud/1122-ingest-m…

…etrics-logs-in-grafana Ingest metrics/logs in Grafana
input-output-hk · Aug 3, 2023 · d4ac66d · d4ac66d
2 parents 1853b2f + 91c4d3e
commit d4ac66d
Show file tree

Hide file tree

Showing 8 changed files with 108 additions and 11 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -453,8 +453,14 @@ jobs:
       ERA_READER_SECRET_KEY: ${{ secrets.ERA_READER_SECRET_KEY }}
       PROMETHEUS_AUTH_USERNAME: ${{ secrets.PROMETHEUS_AUTH_USERNAME }}
       PROMETHEUS_AUTH_PASSWORD: ${{ secrets.PROMETHEUS_AUTH_PASSWORD }}
+      PROMETHEUS_INGEST_HOST: ${{ vars.PROMETHEUS_INGEST_HOST }}
+      PROMETHEUS_INGEST_USERNAME: ${{ secrets.PROMETHEUS_INGEST_USERNAME }}
+      PROMETHEUS_INGEST_PASSWORD: ${{ secrets.PROMETHEUS_INGEST_PASSWORD }}
       LOKI_AUTH_USERNAME: ${{ secrets.LOKI_AUTH_USERNAME }}
       LOKI_AUTH_PASSWORD: ${{ secrets.LOKI_AUTH_PASSWORD }}
+      LOKI_INGEST_HOST: ${{ vars.LOKI_INGEST_HOST }}
+      LOKI_INGEST_USERNAME: ${{ secrets.LOKI_INGEST_USERNAME }}
+      LOKI_INGEST_PASSWORD: ${{ secrets.LOKI_INGEST_PASSWORD }}
 
     defaults:
       run:
@@ -496,8 +502,14 @@ jobs:
           mithril_signers                         = ${{ matrix.mithril_signers }}
           prometheus_auth_username                = "${{ env.PROMETHEUS_AUTH_USERNAME }}"
           prometheus_auth_password                = "${{ env.PROMETHEUS_AUTH_PASSWORD }}"
+          prometheus_ingest_host                  = "${{ env.PROMETHEUS_INGEST_HOST }}"
+          prometheus_ingest_username              = "${{ env.PROMETHEUS_INGEST_USERNAME }}"
+          prometheus_ingest_password              = "${{ env.PROMETHEUS_INGEST_PASSWORD }}"
           loki_auth_username                      = "${{ env.LOKI_AUTH_USERNAME }}"
           loki_auth_password                      = "${{ env.LOKI_AUTH_PASSWORD }}"
+          loki_ingest_host                        = "${{ env.LOKI_INGEST_HOST }}"
+          loki_ingest_username                    = "${{ env.LOKI_INGEST_USERNAME }}"
+          loki_ingest_password                    = "${{ env.LOKI_INGEST_PASSWORD }}"
         EOF
         terraform fmt ./env.variables.tfvars
         cat ./env.variables.tfvars

diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml
@@ -181,8 +181,14 @@ jobs:
       ERA_READER_SECRET_KEY: ${{ secrets.ERA_READER_SECRET_KEY }}
       PROMETHEUS_AUTH_USERNAME: ${{ secrets.PROMETHEUS_AUTH_USERNAME }}
       PROMETHEUS_AUTH_PASSWORD: ${{ secrets.PROMETHEUS_AUTH_PASSWORD }}
+      PROMETHEUS_INGEST_HOST: ${{ vars.PROMETHEUS_INGEST_HOST }}
+      PROMETHEUS_INGEST_USERNAME: ${{ secrets.PROMETHEUS_INGEST_USERNAME }}
+      PROMETHEUS_INGEST_PASSWORD: ${{ secrets.PROMETHEUS_INGEST_PASSWORD }}
       LOKI_AUTH_USERNAME: ${{ secrets.LOKI_AUTH_USERNAME }}
       LOKI_AUTH_PASSWORD: ${{ secrets.LOKI_AUTH_PASSWORD }}
+      LOKI_INGEST_HOST: ${{ vars.LOKI_INGEST_HOST }}
+      LOKI_INGEST_USERNAME: ${{ secrets.LOKI_INGEST_USERNAME }}
+      LOKI_INGEST_PASSWORD: ${{ secrets.LOKI_INGEST_PASSWORD }}
 
     defaults:
       run:
@@ -224,8 +230,14 @@ jobs:
           mithril_signers                         = ${{ matrix.mithril_signers }}
           prometheus_auth_username                = "${{ env.PROMETHEUS_AUTH_USERNAME }}"
           prometheus_auth_password                = "${{ env.PROMETHEUS_AUTH_PASSWORD }}"
+          prometheus_ingest_host                  = "${{ env.PROMETHEUS_INGEST_HOST }}"
+          prometheus_ingest_username              = "${{ env.PROMETHEUS_INGEST_USERNAME }}"
+          prometheus_ingest_password              = "${{ env.PROMETHEUS_INGEST_PASSWORD }}"
           loki_auth_username                      = "${{ env.LOKI_AUTH_USERNAME }}"
           loki_auth_password                      = "${{ env.LOKI_AUTH_PASSWORD }}"
+          loki_ingest_host                        = "${{ env.LOKI_INGEST_HOST }}"
+          loki_ingest_username                    = "${{ env.LOKI_INGEST_USERNAME }}"
+          loki_ingest_password                    = "${{ env.LOKI_INGEST_PASSWORD }}"
         EOF
         terraform fmt ./env.variables.tfvars
         cat ./env.variables.tfvars

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -173,8 +173,14 @@ jobs:
       ERA_READER_SECRET_KEY: ${{ secrets.ERA_READER_SECRET_KEY }}
       PROMETHEUS_AUTH_USERNAME: ${{ secrets.PROMETHEUS_AUTH_USERNAME }}
       PROMETHEUS_AUTH_PASSWORD: ${{ secrets.PROMETHEUS_AUTH_PASSWORD }}
+      PROMETHEUS_INGEST_HOST: ${{ vars.PROMETHEUS_INGEST_HOST }}
+      PROMETHEUS_INGEST_USERNAME: ${{ secrets.PROMETHEUS_INGEST_USERNAME }}
+      PROMETHEUS_INGEST_PASSWORD: ${{ secrets.PROMETHEUS_INGEST_PASSWORD }}
       LOKI_AUTH_USERNAME: ${{ secrets.LOKI_AUTH_USERNAME }}
       LOKI_AUTH_PASSWORD: ${{ secrets.LOKI_AUTH_PASSWORD }}
+      LOKI_INGEST_HOST: ${{ vars.LOKI_INGEST_HOST }}
+      LOKI_INGEST_USERNAME: ${{ secrets.LOKI_INGEST_USERNAME }}
+      LOKI_INGEST_PASSWORD: ${{ secrets.LOKI_INGEST_PASSWORD }}
 
     defaults:
       run:
@@ -217,8 +223,14 @@ jobs:
           mithril_signers                         = ${{ matrix.mithril_signers }}
           prometheus_auth_username                = "${{ env.PROMETHEUS_AUTH_USERNAME }}"
           prometheus_auth_password                = "${{ env.PROMETHEUS_AUTH_PASSWORD }}"
+          prometheus_ingest_host                  = "${{ env.PROMETHEUS_INGEST_HOST }}"
+          prometheus_ingest_username              = "${{ env.PROMETHEUS_INGEST_USERNAME }}"
+          prometheus_ingest_password              = "${{ env.PROMETHEUS_INGEST_PASSWORD }}"
           loki_auth_username                      = "${{ env.LOKI_AUTH_USERNAME }}"
           loki_auth_password                      = "${{ env.LOKI_AUTH_PASSWORD }}"
+          loki_ingest_host                        = "${{ env.LOKI_INGEST_HOST }}"
+          loki_ingest_username                    = "${{ env.LOKI_INGEST_USERNAME }}"
+          loki_ingest_password                    = "${{ env.LOKI_INGEST_PASSWORD }}"
         EOF
         terraform fmt ./env.variables.tfvars
         cat ./env.variables.tfvars

diff --git a/...a/assets/docker/prometheus/prometheus.yml → ...ets/docker/prometheus/prometheus-base.yml b/...a/assets/docker/prometheus/prometheus.yml → ...ets/docker/prometheus/prometheus-base.yml
@@ -27,5 +27,3 @@ scrape_configs:
     file_sd_configs:
     - files:
       - 'cardano.json'
-
-
diff --git a/...ssets/docker/promtail/promtail-config.yml → .../docker/promtail/promtail-config-base.yml b/...ssets/docker/promtail/promtail-config.yml → .../docker/promtail/promtail-config-base.yml
@@ -5,9 +5,6 @@ server:
 positions:
   filename: /tmp/positions.yaml
 
-clients:
-  - url: http://loki:3100/loki/api/v1/push
-
 scrape_configs:
 - job_name: system
   static_configs:
@@ -49,4 +46,7 @@ scrape_configs:
       image_id:
       container_id:
   - output:
-      source: output
+      source: output
+
+clients:
+  - url: http://loki:3100/loki/api/v1/push
diff --git a/mithril-infra/assets/infra.version b/mithril-infra/assets/infra.version
@@ -1 +1 @@
-0.1.9
+0.1.10
diff --git a/mithril-infra/mithril.monitoring.tf b/mithril-infra/mithril.monitoring.tf
@@ -28,6 +28,20 @@ resource "null_resource" "mithril_monitoring" {
       "mkdir -p /home/curry/data/monitoring/prometheus",
       "mkdir -p /home/curry/data/monitoring/loki",
       <<-EOT
+set -e
+# Copy prometheus base configuration
+cp /home/curry/docker/prometheus/prometheus-base.yml /home/curry/docker/prometheus/prometheus.yml
+# Setup prometheus remote write
+if [ -n "${var.prometheus_ingest_host}" ] ; then 
+cat >> /home/curry/docker/prometheus/prometheus.yml << EOF
+
+remote_write:
+- url: https://${var.prometheus_ingest_host}/api/prom/push
+  basic_auth:
+    username: ${var.prometheus_ingest_username}
+    password: ${var.prometheus_ingest_password}
+EOF
+fi
 # Setup prometheus targets configuration for Cardano nodes
 CARDANO_NODES=$(docker ps --format='{{.Names}}:12798,' | grep "cardano-node" | sort | tr -d '\n\t\r ' | sed 's/.$//')
 cat /home/curry/docker/prometheus/cardano.json | jq --arg CARDANO_NODES "$CARDANO_NODES" '. += [{
@@ -38,6 +52,19 @@ cat /home/curry/docker/prometheus/cardano.json | jq --arg CARDANO_NODES "$CARDAN
 }]' | jq '. | map(try(.targets |= split(",")) // .)' > /home/curry/docker/prometheus/cardano.json.new
 rm -f /home/curry/docker/prometheus/cardano.json
 mv /home/curry/docker/prometheus/cardano.json.new docker/prometheus/cardano.json
+EOT
+      ,
+      <<-EOT
+set -e
+# Copy promtail base configuration
+cp /home/curry/docker/promtail/promtail-config-base.yml /home/curry/docker/promtail/promtail-config.yml
+# Setup promtail remote client
+if [ -n "${var.loki_ingest_host}" ] ; then 
+cat >> /home/curry/docker/promtail/promtail-config.yml << EOF
+
+  - url: https://${var.loki_ingest_username}:${var.loki_ingest_password}@${var.loki_ingest_host}/loki/api/v1/push
+EOF
+fi
 EOT
     ]
   }

diff --git a/mithril-infra/variables.tf b/mithril-infra/variables.tf
@@ -174,25 +174,61 @@ variable "mithril_aggregator_auth_password" {
 
 variable "prometheus_auth_username" {
   type        = string
-  description = "The username for authentication on prometheus"
+  description = "The username for authentication on local prometheus endpoint"
   default     = ""
 }
 
 variable "prometheus_auth_password" {
   type        = string
-  description = "The password for authentication on prometheus"
+  description = "The password for authentication on local prometheus endpoint"
+  default     = ""
+}
+
+variable "prometheus_ingest_host" {
+  type        = string
+  description = "The host to ingest on remote prometheus endpoint"
+  default     = ""
+}
+
+variable "prometheus_ingest_username" {
+  type        = string
+  description = "The username to ingest on remote prometheus endpoint"
+  default     = ""
+}
+
+variable "prometheus_ingest_password" {
+  type        = string
+  description = "The password to ingest on remote prometheus endpoint"
   default     = ""
 }
 
 variable "loki_auth_username" {
   type        = string
-  description = "The username for authentication on loki"
+  description = "The username for authentication on local loki endpoint"
   default     = ""
 }
 
 variable "loki_auth_password" {
   type        = string
-  description = "The password for authentication on loki"
+  description = "The password for authentication on local loki endpoint"
+  default     = ""
+}
+
+variable "loki_ingest_host" {
+  type        = string
+  description = "The host to ingest on remote loki endpoint"
+  default     = ""
+}
+
+variable "loki_ingest_username" {
+  type        = string
+  description = "The username to ingest on remote loki endpoint"
+  default     = ""
+}
+
+variable "loki_ingest_password" {
+  type        = string
+  description = "The password to ingest on remote loki endpoint"
   default     = ""
 }