diff --git a/filebeat/README.md b/filebeat/README.md index 576bbf9de..09a0a5a58 100644 --- a/filebeat/README.md +++ b/filebeat/README.md @@ -73,9 +73,11 @@ upgrading to a new chart version. * The default Filebeat configuration file for this chart is configured to use an Elasticsearch endpoint. Without any additional changes, Filebeat will send documents to the service URL that the Elasticsearch Helm chart sets up by -default. You may either set the `ELASTICSEARCH_HOSTS` environment variable in -`extraEnvs` to override this endpoint or modify the default `filebeatConfig` to -change this behavior. +default. The Elasticsearch credentials are also retrieved from +`elasticsearch-master-credentials` Secret from Elasticsearch chart by default. +You may either set the `ELASTICSEARCH_HOSTS`, `ELASTICSEARCH_USER` and +`ELASTICSEARCH_PASSWORD` environment variables in `extraEnvs` to override this +or modify the default `filebeatConfig` to change this behavior. * The default Filebeat configuration file is also configured to capture container logs and enrich them with Kubernetes metadata by default. This will capture all container logs in the cluster. @@ -100,7 +102,7 @@ as a reference. They are also used in the automated testing of this chart. | `daemonset.affinity` | Configurable [affinity][] for filebeat daemonset | `{}` | | `daemonset.enabled` | If true, enable daemonset | `true` | | `daemonset.envFrom` | Templatable string of `envFrom` to be passed to the [environment from variables][] which will be appended to filebeat container for DaemonSet | `[]` | -| `daemonset.extraEnvs` | Extra [environment variables][] which will be appended to filebeat container for DaemonSet | `[]` | +| `daemonset.extraEnvs` | Extra [environment variables][] which will be appended to filebeat container for DaemonSet | see [values.yaml][] | | `daemonset.extraVolumeMounts` | Templatable string of additional `volumeMounts` to be passed to the `tpl` function for DaemonSet | `[]` | | `daemonset.extraVolumes` | Templatable string of additional `volumes` to be passed to the `tpl` function for DaemonSet | `[]` | | `daemonset.hostAliases` | Configurable [hostAliases][] for filebeat DaemonSet | `[]` | @@ -117,7 +119,7 @@ as a reference. They are also used in the automated testing of this chart. | `deployment.affinity` | Configurable [affinity][] for filebeat Deployment | `{}` | | `deployment.enabled` | If true, enable deployment | `false` | | `deployment.envFrom` | Templatable string of `envFrom` to be passed to the [environment from variables][] which will be appended to filebeat container for Deployment | `[]` | -| `deployment.extraEnvs` | Extra [environment variables][] which will be appended to filebeat container for Deployment | `[]` | +| `deployment.extraEnvs` | Extra [environment variables][] which will be appended to filebeat container for Deployment | see [values.yaml][] | | `deployment.extraVolumeMounts` | Templatable string of additional `volumeMounts` to be passed to the `tpl` function for DaemonSet | `[]` | | `deployment.extraVolumes` | Templatable string of additional `volumes` to be passed to the `tpl` function for Deployment | `[]` | | `daemonset.hostAliases` | Configurable [hostAliases][] for filebeat Deployment | `[]` | diff --git a/filebeat/examples/default/test/goss.yaml b/filebeat/examples/default/test/goss.yaml index 43d93354d..4cda3451b 100644 --- a/filebeat/examples/default/test/goss.yaml +++ b/filebeat/examples/default/test/goss.yaml @@ -2,7 +2,7 @@ port: tcp:5066: listening: true ip: - - '127.0.0.1' + - "127.0.0.1" mount: /usr/share/filebeat/data: @@ -28,19 +28,21 @@ http: http://elasticsearch-master:9200/_cat/indices: status: 200 timeout: 2000 + username: "{{ .Env.ELASTICSEARCH_USERNAME }}" + password: "{{ .Env.ELASTICSEARCH_PASSWORD }}" body: - - 'filebeat-8.0.0' + - "filebeat-8.0.0" file: /usr/share/filebeat/filebeat.yml: exists: true contains: - - 'add_kubernetes_metadata' - - 'output.elasticsearch' - - 'elasticsearch-master:9200' + - "add_kubernetes_metadata" + - "output.elasticsearch" + - "elasticsearch-master:9200" command: cd /usr/share/filebeat && filebeat test output: exit-status: 0 stdout: - - 'elasticsearch: http://elasticsearch-master:9200' + - "elasticsearch: http://elasticsearch-master:9200" diff --git a/filebeat/examples/deployment/test/goss.yaml b/filebeat/examples/deployment/test/goss.yaml index b6b86bddb..39dbcaf13 100644 --- a/filebeat/examples/deployment/test/goss.yaml +++ b/filebeat/examples/deployment/test/goss.yaml @@ -2,5 +2,7 @@ http: http://elasticsearch-master:9200/_cat/indices: status: 200 timeout: 2000 + username: "{{ .Env.ELASTICSEARCH_USERNAME }}" + password: "{{ .Env.ELASTICSEARCH_PASSWORD }}" body: - - 'filebeat-8.0.0' + - "filebeat-8.0.0" diff --git a/filebeat/examples/deployment/values.yaml b/filebeat/examples/deployment/values.yaml index bf1cf06c1..f5ba5f295 100644 --- a/filebeat/examples/deployment/values.yaml +++ b/filebeat/examples/deployment/values.yaml @@ -3,14 +3,3 @@ deployment: daemonset: enabled: false - -filebeatConfig: - filebeat.yml: | - filebeat.inputs: - - type: log - paths: - - /usr/share/filebeat/logs/filebeat - - output.elasticsearch: - host: '${NODE_NAME}' - hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}' \ No newline at end of file diff --git a/filebeat/examples/oss/test/goss.yaml b/filebeat/examples/oss/test/goss.yaml index 107b4eb33..3c8301978 100644 --- a/filebeat/examples/oss/test/goss.yaml +++ b/filebeat/examples/oss/test/goss.yaml @@ -2,7 +2,7 @@ port: tcp:5066: listening: true ip: - - '127.0.0.1' + - "127.0.0.1" mount: /usr/share/filebeat/data: @@ -18,5 +18,7 @@ http: http://elasticsearch-master:9200/_cat/indices: status: 200 timeout: 2000 + username: "{{ .Env.ELASTICSEARCH_USERNAME }}" + password: "{{ .Env.ELASTICSEARCH_PASSWORD }}" body: - - 'filebeat-oss-8.0.0' + - "filebeat-oss-8.0.0" diff --git a/filebeat/examples/oss/values.yaml b/filebeat/examples/oss/values.yaml index 7f713fede..ffaf9eb3c 100644 --- a/filebeat/examples/oss/values.yaml +++ b/filebeat/examples/oss/values.yaml @@ -16,6 +16,8 @@ daemonset: output.elasticsearch: host: '${NODE_NAME}' hosts: "elasticsearch-master:9200" + username: '${ELASTICSEARCH_USERNAME}' + password: '${ELASTICSEARCH_PASSWORD}' index: "filebeat-oss-%{[agent.version]}-%{+yyyy.MM.dd}" setup.ilm.enabled: false setup.template.name: "filebeat" diff --git a/filebeat/examples/security/values.yaml b/filebeat/examples/security/values.yaml index 5ed7c5529..6025f5868 100644 --- a/filebeat/examples/security/values.yaml +++ b/filebeat/examples/security/values.yaml @@ -1,37 +1,39 @@ -filebeatConfig: - filebeat.yml: | - filebeat.inputs: - - type: container - paths: - - /var/log/containers/*.log - processors: - - add_kubernetes_metadata: - host: ${NODE_NAME} - matchers: - - logs_path: - logs_path: "/var/log/containers/" +daemonset: + extraEnvs: + - name: "ELASTICSEARCH_HOSTS" + value: "security-master:9200" + - name: "ELASTICSEARCH_USERNAME" + valueFrom: + secretKeyRef: + name: security-master-credentials + key: username + - name: "ELASTICSEARCH_PASSWORD" + valueFrom: + secretKeyRef: + name: security-master-credentials + key: password + filebeatConfig: + filebeat.yml: | + filebeat.inputs: + - type: container + paths: + - /var/log/containers/*.log + processors: + - add_kubernetes_metadata: + host: ${NODE_NAME} + matchers: + - logs_path: + logs_path: "/var/log/containers/" - output.elasticsearch: - username: '${ELASTICSEARCH_USERNAME}' - password: '${ELASTICSEARCH_PASSWORD}' - protocol: https - hosts: ["security-master:9200"] - ssl.certificate_authorities: + output.elasticsearch: + host: '${NODE_NAME}' + hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}' + username: '${ELASTICSEARCH_USERNAME}' + password: '${ELASTICSEARCH_PASSWORD}' + protocol: https + ssl.certificate_authorities: - /usr/share/filebeat/config/certs/elastic-certificate.pem - -secretMounts: - - name: elastic-certificate-pem - secretName: elastic-certificate-pem - path: /usr/share/filebeat/config/certs - -extraEnvs: - - name: 'ELASTICSEARCH_USERNAME' - valueFrom: - secretKeyRef: - name: elastic-credentials - key: username - - name: 'ELASTICSEARCH_PASSWORD' - valueFrom: - secretKeyRef: - name: elastic-credentials - key: password + secretMounts: + - name: elastic-certificate-pem + secretName: elastic-certificate-pem + path: /usr/share/filebeat/config/certs diff --git a/filebeat/examples/upgrade/values.yaml b/filebeat/examples/upgrade/values.yaml index 8b230601e..af3e5bfa4 100644 --- a/filebeat/examples/upgrade/values.yaml +++ b/filebeat/examples/upgrade/values.yaml @@ -1,4 +1,15 @@ --- -extraEnvs: - - name: ELASTICSEARCH_HOSTS - value: upgrade-master:9200 +daemonset: + extraEnvs: + - name: ELASTICSEARCH_HOSTS + value: upgrade-master:9200 + - name: "ELASTICSEARCH_USERNAME" + valueFrom: + secretKeyRef: + name: upgrade-master-credentials + key: username + - name: "ELASTICSEARCH_PASSWORD" + valueFrom: + secretKeyRef: + name: upgrade-master-credentials + key: password diff --git a/filebeat/values.yaml b/filebeat/values.yaml index 8e9daf8da..7e3467748 100755 --- a/filebeat/values.yaml +++ b/filebeat/values.yaml @@ -11,16 +11,25 @@ daemonset: envFrom: [] # - configMapRef: # name: config-secret - extraEnvs: [] - # - name: MY_ENVIRONMENT_VAR - # value: the_value_goes_here + extraEnvs: + - name: "ELASTICSEARCH_USERNAME" + valueFrom: + secretKeyRef: + name: elasticsearch-master-credentials + key: username + - name: "ELASTICSEARCH_PASSWORD" + valueFrom: + secretKeyRef: + name: elasticsearch-master-credentials + key: password + # Allows you to add any config files in /usr/share/filebeat extraVolumes: [] - # - name: extras - # emptyDir: {} + # - name: extras + # emptyDir: {} extraVolumeMounts: [] - # - name: extras - # mountPath: /usr/share/extras - # readOnly: true + # - name: extras + # mountPath: /usr/share/extras + # readOnly: true hostNetworking: false # Allows you to add any config files in /usr/share/filebeat # such as filebeat.yml for daemonset @@ -40,6 +49,8 @@ daemonset: output.elasticsearch: host: '${NODE_NAME}' hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}' + username: '${ELASTICSEARCH_USERNAME}' + password: '${ELASTICSEARCH_PASSWORD}' # Only used when updateStrategy is set to "RollingUpdate" maxUnavailable: 1 nodeSelector: {} @@ -77,9 +88,17 @@ deployment: envFrom: [] # - configMapRef: # name: config-secret - extraEnvs: [] - # - name: MY_ENVIRONMENT_VAR - # value: the_value_goes_here + extraEnvs: + - name: "ELASTICSEARCH_USERNAME" + valueFrom: + secretKeyRef: + name: elasticsearch-master-credentials + key: username + - name: "ELASTICSEARCH_PASSWORD" + valueFrom: + secretKeyRef: + name: elasticsearch-master-credentials + key: password # Allows you to add any config files in /usr/share/filebeat extraVolumes: [] # - name: extras @@ -92,13 +111,15 @@ deployment: filebeatConfig: filebeat.yml: | filebeat.inputs: - - type: tcp - max_message_size: 10MiB - host: "localhost:9000" + - type: log + paths: + - /usr/share/filebeat/logs/filebeat output.elasticsearch: - host: '${NODE_NAME}' - hosts: '${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}' + host: "${NODE_NAME}" + hosts: "${ELASTICSEARCH_HOSTS:elasticsearch-master:9200}" + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" nodeSelector: {} # A list of secrets and their paths to mount inside the pod # This is useful for mounting certificates for security other sensitive values @@ -180,27 +201,26 @@ readinessProbe: managedServiceAccount: true clusterRoleRules: -- apiGroups: - - "" - resources: - - namespaces - - nodes - - pods - verbs: - - get - - list - - watch + - apiGroups: + - "" + resources: + - namespaces + - nodes + - pods + verbs: + - get + - list + - watch podAnnotations: {} - # iam.amazonaws.com/role: es-cluster +# iam.amazonaws.com/role: es-cluster # Custom service account override that the pod will use serviceAccount: "" # Annotations to add to the ServiceAccount that is created if the serviceAccount value isn't set. serviceAccountAnnotations: {} - - # eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount +# eks.amazonaws.com/role-arn: arn:aws:iam::111111111111:role/k8s.clustername.namespace.serviceaccount # How long to wait for Filebeat pods to stop gracefully terminationGracePeriod: 30