Feat/add-victoria-metrics

kubernetes/main/apps/home/home-assistant/app/helmrelease.yaml

@@ -118,15 +118,15 @@ spec:
         enabled: true
         className: internal
         hosts:
-          - host: &host hass-code.rafaribe.com
+          - host: &host2 hass-code.rafaribe.com
             paths:
               - path: /
                 service:
                   name: main
                   port: code
         tls:
           - hosts:
-              - *host
+              - *host2
     persistence:
       config:
         enabled: true

kubernetes/main/apps/observability/alertmanager/app/config/alertmanager.yaml

@@ -0,0 +1,61 @@
+receivers:
+  - name: "null"
+  - name: "discord"
+    webhook_configs:
+      - url: "http://alertmanager-discord.observability.svc.cluster.local:9094"
+  - name: "pushover"
+    pushover_configs:
+      - html: true
+        token_file: /etc/secrets/pushover_api_token
+        user_key_file: /etc/secrets/pushover_api_userkey
+        send_resolved: true
+        priority: |-
+          {{ if eq .Status "firing" }}1{{ else }}0{{ end }}
+        url_title: View in Alert Manager
+        title: |-
+          [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }}
+        message: |-
+          {{- range .Alerts }}
+            {{- if ne .Labels.severity "" }}
+              <b>Severity:</b> <i>{{ .Labels.severity }}</i>
+            {{- else }}
+              <b>Severity:</b> <i>N/A</i>
+            {{- end }}
+            {{- if ne .Annotations.description "" }}
+              <b>Description:</b> <i>{{ .Annotations.description }}</i>
+            {{- else if ne .Annotations.summary "" }}
+              <b>Summary:</b> <i>{{ .Annotations.summary }}</i>
+            {{- else if ne .Annotations.message "" }}
+              <b>Message:</b> <i>{{ .Annotations.message }}</i>
+            {{- else }}
+              <b>Description:</b> <i>N/A</i>
+            {{- end }}
+            {{- if gt (len .Labels.SortedPairs) 0 }}
+              <b>Details:</b>
+              {{- range .Labels.SortedPairs }}
+                • <b>{{ .Name }}:</b> <i>{{ .Value }}</i>
+              {{- end }}
+            {{- end }}
+          {{- end }}
+
+route:
+  group_by: ["alertname", "job"]
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 6h
+  receiver: "pushover"
+  routes:
+    - receiver: "null"
+      matchers:
+        - alertname =~ "InfoInhibitor|Watchdog"
+    - receiver: "pushover"
+      matchers:
+        - severity = critical
+      continue: true
+
+inhibit_rules:
+  - source_matchers:
+      - severity = "critical"
+    target_matchers:
+      - severity = "warning"
+    equal: ["alertname", "namespace"]

kubernetes/main/apps/observability/alertmanager/app/externalsecret.yaml

@@ -0,0 +1,21 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: &name alertmanager-secret
+  namespace: monitoring
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: akeyless-secret-store
+  target:
+    name: alertmanager-secret
+    creationPolicy: Owner
+    deletionPolicy: "Delete"
+    template:
+      engineVersion: v2
+      data:
+        PUSHOVER_API_USERKEY: "{{ .PUSHOVER_API_USERKEY }}"
+        PUSHOVER_API_TOKEN: "{{ .PUSHOVER_API_TOKEN }}"
+  dataFrom:
+    - extract:
+        key: /webhook_discord

kubernetes/main/apps/observability/alertmanager/app/helmrelease.yaml

@@ -0,0 +1,89 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2.schema.json
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: alertmanager
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: app-template
+      version: 3.2.1
+      interval: 30m
+      sourceRef:
+        kind: HelmRepository
+        name: bjw-s
+        namespace: flux-system
+  values:
+    controllers:
+      alertmanager:
+        type: statefulset
+        annotations:
+          reloader.stakater.com/auto: "true"
+
+        statefulset:
+          volumeClaimTemplates:
+            - name: storage
+              accessMode: ReadWriteOnce
+              size: 50Mi
+              storageClass: ceph-block
+              globalMounts:
+                - path: /alertmanager
+
+        containers:
+          alertmanager:
+            image:
+              repository: quay.io/prometheus/alertmanager
+              tag: v0.27.0
+            ports:
+              - name: http
+                containerPort: 9093
+            probes:
+              liveness:
+                enabled: true
+              readiness:
+                enabled: true
+              startup:
+                enabled: true
+                spec:
+                  failureThreshold: 30
+                  periodSeconds: 5
+            resources:
+              requests:
+                cpu: 11m
+                memory: 50M
+              limits:
+                memory: 99M
+
+    service:
+      app:
+        controller: alertmanager
+        ports:
+          http:
+            port: 9093
+
+    ingress:
+      app:
+        className: internal
+        annotations:
+          external-dns.alpha.kubernetes.io/target: ingress-int.rafaribe.com
+        hosts:
+          - host: alertmanager.rafaribe.com
+            paths:
+              - path: /
+                service:
+                  identifier: app
+                  port: http
+
+    persistence:
+      config:
+        type: configMap
+        name: alertmanager-configmap
+        globalMounts:
+          - path: /etc/alertmanager
+      secrets:
+        type: secret
+        name: alertmanager-secret
+        globalMounts:
+          - path: /etc/secrets

kubernetes/main/apps/observability/thanos/app/kustomization.yaml → kubernetes/main/apps/observability/alertmanager/app/kustomization.yaml

@@ -2,12 +2,15 @@
 # yaml-language-server: $schema=https://json.schemastore.org/kustomization
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
+namespace: monitoring
 resources:
-  - ./objectbucketclaim.yaml
+  - ./externalsecret.yaml
   - ./helmrelease.yaml
 configMapGenerator:
-  - name: thanos-cache-configmap
+  - name: alertmanager-configmap
     files:
-      - cache.yaml=./resources/cache.yaml
+      - ./config/alertmanager.yaml
 generatorOptions:
   disableNameSuffixHash: true
+  annotations:
+    kustomize.toolkit.fluxcd.io/substitute: disabled

kubernetes/main/apps/observability/alertmanager/ks.yaml

@@ -0,0 +1,19 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &appname alertmanager
+  namespace: flux-system
+spec:
+  targetNamespace: observability
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *appname
+  interval: 10m
+  path: "./kubernetes/main/apps/observability/alertmanager/app/"
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: home-ops
+  wait: true