From 41c0e9102ffeedf2472c3d202698873cc01b27c9 Mon Sep 17 00:00:00 2001 From: Tuan Anh Nguyen <48820639+tuananhnguyen-ct@users.noreply.github.com> Date: Fri, 5 Feb 2021 17:21:16 +0700 Subject: [PATCH] [logstash] Add support to use pattern files (#883) Co-authored-by: Julien Mailleret <8582351+jmlrt@users.noreply.github.com> --- logstash/README.md | 2 ++ logstash/templates/configmap-pattern.yaml | 17 +++++++++++++++++ logstash/templates/statefulset.yaml | 16 ++++++++++++++++ logstash/tests/logstash_test.py | 19 +++++++++++++++++++ logstash/values.yaml | 6 ++++++ 5 files changed, 60 insertions(+) create mode 100644 logstash/templates/configmap-pattern.yaml diff --git a/logstash/README.md b/logstash/README.md index 3c5c2c5bb..feb5cca6f 100644 --- a/logstash/README.md +++ b/logstash/README.md @@ -129,6 +129,8 @@ useful for the [http input plugin][], for instance. | `logstashConfig` | Allows you to add any config files in `/usr/share/logstash/config/` such as `logstash.yml` and `log4j2.properties` See [values.yaml][] for an example of the formatting | `{}` | | `logstashJavaOpts` | Java options for Logstash. This is where you should configure the JVM heap size | `-Xmx1g -Xms1g` | | `logstashPipeline` | Allows you to add any pipeline files in `/usr/share/logstash/pipeline/` | `{}` | +| `logstashPatternDir` | Allows you to define a custom directory to store patten files | `/usr/share/logstash/patterns/` | +| `logstashPattern` | Allows you to add any pattern files in `logstashPatternDir` | `{}` | | `maxUnavailable` | The [maxUnavailable][] value for the pod disruption budget. By default this will prevent Kubernetes from having more than 1 unhealthy pod in the node group | `1` | | `nameOverride` | Overrides the chart name for resources. If not set the name will default to `.Chart.Name` | `""` | | `nodeAffinity` | Value for the [node affinity settings][] | `{}` | diff --git a/logstash/templates/configmap-pattern.yaml b/logstash/templates/configmap-pattern.yaml new file mode 100644 index 000000000..0eb58593d --- /dev/null +++ b/logstash/templates/configmap-pattern.yaml @@ -0,0 +1,17 @@ +{{- if .Values.logstashPattern }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "logstash.fullname" . }}-pattern + labels: + app: "{{ template "logstash.fullname" . }}" + chart: "{{ .Chart.Name }}" + heritage: {{ .Release.Service | quote }} + release: {{ .Release.Name | quote }} +data: +{{- range $path, $config := .Values.logstashPattern }} + {{ $path }}: | +{{ tpl $config $ | indent 4 -}} +{{- end -}} +{{- end -}} diff --git a/logstash/templates/statefulset.yaml b/logstash/templates/statefulset.yaml index 7f17586b4..de511c03f 100644 --- a/logstash/templates/statefulset.yaml +++ b/logstash/templates/statefulset.yaml @@ -55,6 +55,9 @@ spec: {{- if .Values.logstashPipeline }} pipelinechecksum: {{ include (print .Template.BasePath "/configmap-pipeline.yaml") . | sha256sum | trunc 63 }} {{- end }} + {{- if .Values.logstashPattern }} + patternchecksum: {{ include (print .Template.BasePath "/configmap-pattern.yaml") . | sha256sum | trunc 63 }} + {{- end }} {{- if .Values.secrets }} secretschecksum: {{ include (print .Template.BasePath "/secret.yaml") . | sha256sum | trunc 63 }} {{- end }} @@ -127,6 +130,11 @@ spec: configMap: name: {{ template "logstash.fullname" . }}-pipeline {{- end }} + {{- if .Values.logstashPattern }} + - name: logstashpattern + configMap: + name: {{ template "logstash.fullname" . }}-pattern + {{- end }} {{- if .Values.extraVolumes }} {{ tpl .Values.extraVolumes . | indent 8 }} {{- end }} @@ -191,6 +199,14 @@ spec: mountPath: /usr/share/logstash/pipeline/{{ $path }} subPath: {{ $path }} {{- end -}} + {{- if .Values.logstashPattern }} + {{- $logstashPatternDir := .Values.logstashPatternDir -}} + {{- range $path, $config := .Values.logstashPattern }} + - name: logstashpattern + mountPath: {{ $logstashPatternDir }}{{ $path }} + subPath: {{ $path }} + {{- end -}} + {{- end -}} {{- if .Values.extraVolumeMounts }} {{ tpl .Values.extraVolumeMounts . | indent 10 }} {{- end }} diff --git a/logstash/tests/logstash_test.py b/logstash/tests/logstash_test.py index ebb679b98..6c76501cb 100755 --- a/logstash/tests/logstash_test.py +++ b/logstash/tests/logstash_test.py @@ -622,6 +622,25 @@ def test_adding_in_pipeline(): ) +def test_adding_in_pattern(): + config = """ +logstashPattern: + pattern.conf: | + DPKG_VERSION [-+~<>\.0-9a-zA-Z]+ +""" + r = helm_template(config) + c = r["configmap"][name + "-pattern"]["data"] + + assert "pattern.conf" in c + + assert "DPKG_VERSION [-+~<>\.0-9a-zA-Z]+" in c["pattern.conf"] + + assert ( + "patternchecksum" + in r["statefulset"][name]["spec"]["template"]["metadata"]["annotations"] + ) + + def test_priority_class_name(): config = """ priorityClassName: "" diff --git a/logstash/values.yaml b/logstash/values.yaml index a5fb6d010..0751f9521 100755 --- a/logstash/values.yaml +++ b/logstash/values.yaml @@ -25,6 +25,12 @@ logstashPipeline: {} # } # output { stdout { } } +# Allows you to add any pattern files in your custom pattern dir +logstashPatternDir: "/usr/share/logstash/patterns/" +logstashPattern: {} +# pattern.conf: | +# DPKG_VERSION [-+~<>\.0-9a-zA-Z]+ + # Extra environment variables to append to this nodeGroup # This will be appended to the current 'env:' key. You can use any of the kubernetes env # syntax here