helm: enable hot tier and Nginx parser (parseablehq#1061)

parmesant · Jan 13, 2025 · 8554d1f · 8554d1f
1 parent 1a9ef45
commit 8554d1f
Show file tree

Hide file tree

Showing 2 changed files with 39 additions and 2 deletions.
diff --git a/helm/templates/querier-statefulset.yaml b/helm/templates/querier-statefulset.yaml
@@ -52,6 +52,10 @@ spec:
                 fieldPath: metadata.name
           - name: P_MODE
             value: "query"
+          {{- if .Values.parseable.persistence.querier.enabled }}
+          - name: P_HOT_TIER_DIR
+            value: "/parseable/hot-tier"
+          {{- end }}
           {{- range $key, $value :=  .Values.parseable.env }}
           - name: {{ $key }}
             value: {{ tpl $value $ | quote }}

diff --git a/helm/values.yaml b/helm/values.yaml
@@ -5,7 +5,7 @@ parseable:
     pullPolicy: Always
   ## object store can be local-store, s3-store, blob-store or gcs-store.
   store: local-store
-  ## Set to true if you want to deploy Parseable in a HA mode (multiple ingestors)
+  ## Set to true if you want to deploy Parseable in a HA mode (multiple ingestors + hot tier)
   ## Please note that highAvailability is not supported in local mode
   highAvailability:
     enabled: false
@@ -66,11 +66,13 @@ parseable:
       storageClass: ""
       accessMode: ReadWriteOnce
       size: 5Gi
+  ## enabling the disk on querier adds the hot-tier.
+  ## local-store is not supported on hot-tier.
     querier:
       enabled: false
       storageClass: ""
       accessMode: ReadWriteOnce
-      size: 5Gi
+      size: 20Gi
   ## comment out the secrets depending upon deployment option
   ## localModeSecret if store is set to local
   ## blobModeSecret if store is set to blob-store
@@ -377,6 +379,22 @@ fluent-bit:
           Mem_Buf_Limit 5MB
           Skip_Long_Lines On
 
+      # [INPUT]
+      #     Name tail
+      #     Path /var/log/containers/{{ NGINX_POD_NAME }}-*.log
+      #     Parser nginx_access
+      #     Tag kube.*
+      #     Mem_Buf_Limit 5MB
+      #     Skip_Long_Lines On
+
+      # [INPUT]
+      #     Name tail
+      #     Path /var/log/containers/{{ NGINX_POD_NAME }}-*.log
+      #     Parser nginx_error
+      #     Tag kube.*
+      #     Mem_Buf_Limit 5MB
+      #     Skip_Long_Lines On
+
       [INPUT]
           name kubernetes_events
           tag k8s_events
@@ -427,6 +445,21 @@ fluent-bit:
           Time_Keep Off
           Time_Key time
           Time_Format %Y-%m-%dT%H:%M:%S.%L
+
+      # [PARSER]
+      #     Name     nginx_error
+      #     Format   regex
+      #     Regex    ^(?<timestamp>\S+Z) stderr F (?<error_timestamp>\S+ \S+) \[(?<log_level>\S+)\] (?<process_id>\d+#\d+): \*(?<request_id>\d+) (?<error_message>.*?) client: (?<client_ip>\S+), server: (?<server_name>\S+), request: "(?<request_method>\S+) (?<request_path>\S+) HTTP/\S+", upstream: "(?<upstream_url>[^"]+)", host: "(?<host>\S+)"$
+      #     Time_Key  timestamp
+      #     Time_Format %Y/%m/%d %H:%M:%S
+      
+      # [PARSER]
+      #     Name   nginx_access
+      #     Format regex
+      #     Regex  (?<remote_addr>[^ ]*) - (?<remote_user>[^ ]*) \[(?<timestamp>[^\]]*)\] "(?<method>\S+)(?: +(?<request>[^\"]*?)(?: +\S*)?)?" (?<status>[^ ]*) (?<body_bytes_sent>[^ ]*) "(?<http_referer>[^\"]*)" "(?<http_user_agent>[^\"]*)" (?<request_length>[^ ]*) (?<request_time>[^ ]*) \[(?<proxy_upstream_name>[^ ]*)\] \[(?<proxy_alternative_upstream_name>[^ ]*)\] (?<upstream_addr>[^,]*),?(?:[^,]*),?(?:[^ ]*) (?<upstream_response_length>[^,]*),?(?:[^,]*),?(?:[^ ]*) (?<upstream_response_time>[^,]*),?(?:[^,]*),?(?:[^ ]*) (?<upstream_status>[^,]*),?(?:[^,]*),?(?:[^ ]*) (?<req_id>[^ ]*)
+      #     Time_Key timestamp
+      #     Time_Format %d/%b/%Y:%H:%M:%S %z
+
   # The config volume is mounted by default, either to the existingConfigMap value, or the default of "fluent-bit.fullname"
   volumeMounts:
     - name: config