[filebeat] use new elasticsearch credentials

This commit updates filebeat values to use the new Elasticsearch credentials from #1384. Relates to #1375
elastic · Oct 12, 2021 · 35fc6b3 · 35fc6b3
1 parent 97c0e58
commit 35fc6b3
Show file tree

Hide file tree

Showing 9 changed files with 122 additions and 91 deletions.
diff --git a/filebeat/README.md b/filebeat/README.md
@@ -73,9 +73,11 @@ upgrading to a new chart version.
 * The default Filebeat configuration file for this chart is configured to use an
 Elasticsearch endpoint. Without any additional changes, Filebeat will send
 documents to the service URL that the Elasticsearch Helm chart sets up by
-default. You may either set the `ELASTICSEARCH_HOSTS` environment variable in
-`extraEnvs` to override this endpoint or modify the default `filebeatConfig` to
-change this behavior.
+default. The Elasticsearch credentials are also retrieved from
+`elasticsearch-master-credentials` Secret from Elasticsearch chart by default.
+You may either set the `ELASTICSEARCH_HOSTS`, `ELASTICSEARCH_USER` and
+`ELASTICSEARCH_PASSWORD` environment variables in `extraEnvs` to override this
+or modify the default `filebeatConfig` to change this behavior.
 * The default Filebeat configuration file is also configured to capture
 container logs and enrich them with Kubernetes metadata by default. This will
 capture all container logs in the cluster.
@@ -100,7 +102,7 @@ as a reference. They are also used in the automated testing of this chart.
 | `daemonset.affinity`           | Configurable [affinity][] for filebeat daemonset                                                                                                                             | `{}`                               |
 | `daemonset.enabled`            | If true, enable daemonset                                                                                                                                                    | `true`                             |
 | `daemonset.envFrom`            | Templatable string of `envFrom` to be passed to the  [environment from variables][] which will be appended to filebeat container for DaemonSet                               | `[]`                               |
-| `daemonset.extraEnvs`          | Extra [environment variables][] which will be appended to filebeat container for DaemonSet                                                                                   | `[]`                               |
+| `daemonset.extraEnvs`          | Extra [environment variables][] which will be appended to filebeat container for DaemonSet                                                                                   | see [values.yaml][]                |
 | `daemonset.extraVolumeMounts`  | Templatable string of additional `volumeMounts` to be passed to the `tpl` function for DaemonSet                                                                             | `[]`                               |
 | `daemonset.extraVolumes`       | Templatable string of additional `volumes` to be passed to the `tpl` function for DaemonSet                                                                                  | `[]`                               |
 | `daemonset.hostAliases`        | Configurable [hostAliases][] for filebeat DaemonSet                                                                                                                          | `[]`                               |
@@ -117,7 +119,7 @@ as a reference. They are also used in the automated testing of this chart.
 | `deployment.affinity`          | Configurable [affinity][] for filebeat Deployment                                                                                                                            | `{}`                               |
 | `deployment.enabled`           | If true, enable deployment                                                                                                                                                   | `false`                            |
 | `deployment.envFrom`           | Templatable string of `envFrom` to be passed to the  [environment from variables][] which will be appended to filebeat container for Deployment                              | `[]`                               |
-| `deployment.extraEnvs`         | Extra [environment variables][] which will be appended to filebeat container for Deployment                                                                                  | `[]`                               |
+| `deployment.extraEnvs`         | Extra [environment variables][] which will be appended to filebeat container for Deployment                                                                                  | see [values.yaml][]                |
 | `deployment.extraVolumeMounts` | Templatable string of additional `volumeMounts` to be passed to the `tpl` function for DaemonSet                                                                             | `[]`                               |
 | `deployment.extraVolumes`      | Templatable string of additional `volumes` to be passed to the `tpl` function for Deployment                                                                                 | `[]`                               |
 | `daemonset.hostAliases`        | Configurable [hostAliases][] for filebeat Deployment                                                                                                                         | `[]`                               |

diff --git a/filebeat/examples/default/test/goss.yaml b/filebeat/examples/default/test/goss.yaml
@@ -2,7 +2,7 @@ port:
   tcp:5066:
     listening: true
     ip:
-    - '127.0.0.1'
+      - "127.0.0.1"
 
 mount:
   /usr/share/filebeat/data:
@@ -28,19 +28,21 @@ http:
   http://elasticsearch-master:9200/_cat/indices:
     status: 200
     timeout: 2000
+    username: "{{ .Env.ELASTICSEARCH_USERNAME }}"
+    password: "{{ .Env.ELASTICSEARCH_PASSWORD }}"
     body:
-      - 'filebeat-8.0.0'
+      - "filebeat-8.0.0"
 
 file:
   /usr/share/filebeat/filebeat.yml:
     exists: true
     contains:
-      - 'add_kubernetes_metadata'
-      - 'output.elasticsearch'
-      - 'elasticsearch-master:9200'
+      - "add_kubernetes_metadata"
+      - "output.elasticsearch"
+      - "elasticsearch-master:9200"
 
 command:
   cd /usr/share/filebeat && filebeat test output:
     exit-status: 0
     stdout:
-      - 'elasticsearch: http://elasticsearch-master:9200'
+      - "elasticsearch: http://elasticsearch-master:9200"
diff --git a/filebeat/examples/deployment/test/goss.yaml b/filebeat/examples/deployment/test/goss.yaml
@@ -2,5 +2,7 @@ http:
   http://elasticsearch-master:9200/_cat/indices:
     status: 200
     timeout: 2000
+    username: "{{ .Env.ELASTICSEARCH_USERNAME }}"
+    password: "{{ .Env.ELASTICSEARCH_PASSWORD }}"
     body:
-      - 'filebeat-8.0.0'
+      - "filebeat-8.0.0"
diff --git a/filebeat/examples/deployment/values.yaml b/filebeat/examples/deployment/values.yaml
@@ -3,14 +3,3 @@ deployment:
 
 daemonset:
   enabled: false
-
-filebeatConfig:
-  filebeat.yml: |
-    filebeat.inputs:
-      - type: log 
-        paths:
-        - /usr/share/filebeat/logs/filebeat
-
-    output.elasticsearch:
-      host: '${NODE_NAME}'
-      hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}'
diff --git a/filebeat/examples/oss/test/goss.yaml b/filebeat/examples/oss/test/goss.yaml
@@ -2,7 +2,7 @@ port:
   tcp:5066:
     listening: true
     ip:
-    - '127.0.0.1'
+      - "127.0.0.1"
 
 mount:
   /usr/share/filebeat/data:
@@ -18,5 +18,7 @@ http:
   http://elasticsearch-master:9200/_cat/indices:
     status: 200
     timeout: 2000
+    username: "{{ .Env.ELASTICSEARCH_USERNAME }}"
+    password: "{{ .Env.ELASTICSEARCH_PASSWORD }}"
     body:
-      - 'filebeat-oss-8.0.0'
+      - "filebeat-oss-8.0.0"
diff --git a/filebeat/examples/oss/values.yaml b/filebeat/examples/oss/values.yaml
@@ -16,6 +16,8 @@ daemonset:
       output.elasticsearch:
         host: '${NODE_NAME}'
         hosts: "elasticsearch-master:9200"
+        username: '${ELASTICSEARCH_USERNAME}'
+        password: '${ELASTICSEARCH_PASSWORD}'
         index: "filebeat-oss-%{[agent.version]}-%{+yyyy.MM.dd}"
       setup.ilm.enabled: false
       setup.template.name: "filebeat"

diff --git a/filebeat/examples/security/values.yaml b/filebeat/examples/security/values.yaml
@@ -1,37 +1,38 @@
-filebeatConfig:
-  filebeat.yml: |
-    filebeat.inputs:
-    - type: container
-      paths:
-        - /var/log/containers/*.log
-      processors:
-      - add_kubernetes_metadata:
-          host: ${NODE_NAME}
-          matchers:
-          - logs_path:
-              logs_path: "/var/log/containers/"
+daemonset:
+  extraEnvs:
+    - name: "ELASTICSEARCH_HOSTS"
+      value: "security-master:9200"
+    - name: "ELASTICSEARCH_USERNAME"
+      valueFrom:
+        secretKeyRef:
+          name: security-master-credentials
+          key: username
+    - name: "ELASTICSEARCH_PASSWORD"
+      valueFrom:
+        secretKeyRef:
+          name: security-master-credentials
+          key: password
+  filebeatConfig:
+    filebeat.yml: |
+      filebeat.inputs:
+      - type: container
+        paths:
+          - /var/log/containers/*.log
+        processors:
+        - add_kubernetes_metadata:
+            host: ${NODE_NAME}
+            matchers:
+            - logs_path:
+                logs_path: "/var/log/containers/"
 
-    output.elasticsearch:
-      username: '${ELASTICSEARCH_USERNAME}'
-      password: '${ELASTICSEARCH_PASSWORD}'
-      protocol: https
-      hosts: ["security-master:9200"]
+      output.elasticsearch:
+        host: '${NODE_NAME}'
+        hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}'
+        username: '${ELASTICSEARCH_USERNAME}'
+        password: '${ELASTICSEARCH_PASSWORD}'
       ssl.certificate_authorities:
         - /usr/share/filebeat/config/certs/elastic-certificate.pem
-
-secretMounts:
-  - name: elastic-certificate-pem
-    secretName: elastic-certificate-pem
-    path: /usr/share/filebeat/config/certs
-
-extraEnvs:
-  - name: 'ELASTICSEARCH_USERNAME'
-    valueFrom:
-      secretKeyRef:
-        name: elastic-credentials
-        key: username
-  - name: 'ELASTICSEARCH_PASSWORD'
-    valueFrom:
-      secretKeyRef:
-        name: elastic-credentials
-        key: password
+  secretMounts:
+    - name: elastic-certificate-pem
+      secretName: elastic-certificate-pem
+      path: /usr/share/filebeat/config/certs
diff --git a/filebeat/examples/upgrade/values.yaml b/filebeat/examples/upgrade/values.yaml
@@ -1,4 +1,15 @@
 ---
-extraEnvs:
-  - name: ELASTICSEARCH_HOSTS
-    value: upgrade-master:9200
+daemonset:
+  extraEnvs:
+    - name: ELASTICSEARCH_HOSTS
+      value: upgrade-master:9200
+    - name: "ELASTICSEARCH_USERNAME"
+      valueFrom:
+        secretKeyRef:
+          name: upgrade-master-credentials
+          key: username
+    - name: "ELASTICSEARCH_PASSWORD"
+      valueFrom:
+        secretKeyRef:
+          name: upgrade-master-credentials
+          key: password
diff --git a/filebeat/values.yaml b/filebeat/values.yaml
@@ -11,16 +11,25 @@ daemonset:
   envFrom: []
   # - configMapRef:
   #     name: config-secret
-  extraEnvs: []
-  #  - name: MY_ENVIRONMENT_VAR
-  #    value: the_value_goes_here
+  extraEnvs:
+    - name: "ELASTICSEARCH_USERNAME"
+      valueFrom:
+        secretKeyRef:
+          name: elasticsearch-master-credentials
+          key: username
+    - name: "ELASTICSEARCH_PASSWORD"
+      valueFrom:
+        secretKeyRef:
+          name: elasticsearch-master-credentials
+          key: password
+  # Allows you to add any config files in /usr/share/filebeat
   extraVolumes: []
-    # - name: extras
-    #   emptyDir: {}
+  # - name: extras
+  #   emptyDir: {}
   extraVolumeMounts: []
-    # - name: extras
-    #   mountPath: /usr/share/extras
-    #   readOnly: true
+  # - name: extras
+  #   mountPath: /usr/share/extras
+  #   readOnly: true
   hostNetworking: false
   # Allows you to add any config files in /usr/share/filebeat
   # such as filebeat.yml for daemonset
@@ -40,6 +49,8 @@ daemonset:
       output.elasticsearch:
         host: '${NODE_NAME}'
         hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}'
+        username: '${ELASTICSEARCH_USERNAME}'
+        password: '${ELASTICSEARCH_PASSWORD}'
   # Only used when updateStrategy is set to "RollingUpdate"
   maxUnavailable: 1
   nodeSelector: {}
@@ -77,9 +88,17 @@ deployment:
   envFrom: []
   # - configMapRef:
   #     name: config-secret
-  extraEnvs: []
-  #  - name: MY_ENVIRONMENT_VAR
-  #    value: the_value_goes_here
+  extraEnvs:
+    - name: "ELASTICSEARCH_USERNAME"
+      valueFrom:
+        secretKeyRef:
+          name: elasticsearch-master-credentials
+          key: username
+    - name: "ELASTICSEARCH_PASSWORD"
+      valueFrom:
+        secretKeyRef:
+          name: elasticsearch-master-credentials
+          key: password
   # Allows you to add any config files in /usr/share/filebeat
   extraVolumes: []
   # - name: extras
@@ -92,13 +111,15 @@ deployment:
   filebeatConfig:
     filebeat.yml: |
       filebeat.inputs:
-      - type: tcp
-        max_message_size: 10MiB
-        host: "localhost:9000"
+        - type: log
+          paths:
+            - /usr/share/filebeat/logs/filebeat
 
       output.elasticsearch:
-        host: '${NODE_NAME}'
-        hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}'
+        host: "${NODE_NAME}"
+        hosts: "${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}"
+        username: "${ELASTICSEARCH_USERNAME}"
+        password: "${ELASTICSEARCH_PASSWORD}"
   nodeSelector: {}
   # A list of secrets and their paths to mount inside the pod
   # This is useful for mounting certificates for security other sensitive values
@@ -180,27 +201,26 @@ readinessProbe:
 managedServiceAccount: true
 
 clusterRoleRules:
-- apiGroups:
-  - ""
-  resources:
-  - namespaces
-  - nodes
-  - pods
-  verbs:
-  - get
-  - list
-  - watch
+  - apiGroups:
+      - ""
+    resources:
+      - namespaces
+      - nodes
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
 
 podAnnotations: {}
-  # iam.amazonaws.com/role: es-cluster
+# iam.amazonaws.com/role: es-cluster
 
 # Custom service account override that the pod will use
 serviceAccount: ""
 
 # Annotations to add to the ServiceAccount that is created if the serviceAccount value isn't set.
 serviceAccountAnnotations: {}
-
-  # eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount
+# eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount
 
 # How long to wait for Filebeat pods to stop gracefully
 terminationGracePeriod: 30