FIX: Update fluent-bit version to 2.1.3 (#240)

* FIX: Update fluent-bit version to 2.1.3
FIX: Fix kubernetes deployment

* Update k8s-manifest/README.md to include namespace
in delete commands

logs/CHANGELOG.md

@@ -1,6 +1,10 @@
 # Changelog
 
-## Fluentd-http
+## Fluentd
+
+### v1.15.2 / 2023-06-20
+
+* [DOWNGRADE] Restoring the image version to 0.0.7 in the Fluentd Helm (Coralogix Plugin) 'values.yaml' file
 
 ### v1.15.2 / 2023-06-16
 
@@ -24,6 +28,15 @@
 
 ## Fluent-Bit
 
+### v2.1.3 / 2023-06-21
+
+* [UPGRADE] Upgrade Fluent-bit version to 2.1.3
+* [UPGRADE] Upgrade upstream Fluent-Bit helm chart dependency version to 0.30.4
+* [CHANGE] Updated version scheme to map to upstream version
+* [FIX] Updated Helm documentation to ensure deployment to "monitoring" namespace
+* [FIX] Updated kubernetes deployment luascript to avoid excessive errors in pod logs
+* [FIX] Updated kubernetes daemonset to consume fluent-bit-env configmap
+
 ### v0.1.0 / 2023-02-03
 
 * [UPGRADE] Upgrade Fluent-bit version to 2.0.8
@@ -42,7 +55,3 @@
 * [UPGRADE] Upgrade Fluent-Bit version to 1.9.3
 * [CHANGE] Set Retry_Limit to False [no limit] to keep retrying send the logs and not lose any data
   ([#48](https://github.com/coralogix/eng-integrations/pull/48))
-
-## Fluentd-http
-
-* [DOWNGRADE] Restoring the image version to 0.0.7 in the 'values.yaml' file

logs/fluent-bit/image/Dockerfile

@@ -8,10 +8,10 @@ COPY ./out_coralogix.go ./go.mod ./go.sum ./
 RUN go mod vendor && \
     go build -buildmode=c-shared -ldflags "-s -w" -mod=vendor -o out_coralogix.$TARGETARCH.so .
 
-FROM fluent/fluent-bit:2.0.8
+FROM fluent/fluent-bit:2.1.3
 ARG TARGETARCH
 MAINTAINER Coralogix Inc. <info@coralogix.com>
-LABEL Description="Special Fluent-Bit image for Coralogix integration" Vendor="Coralogix Inc." Version="1.0.0"
+LABEL Description="Special Fluent-Bit image for Coralogix integration" Vendor="Coralogix Inc." Version="2.1.3"
 COPY --from=builder /go/src/app/plugins.conf /fluent-bit/etc/
 COPY --from=builder /go/src/app/out_coralogix.$TARGETARCH.so /fluent-bit/plugins/out_coralogix.so
 COPY ./functions.lua /fluent-bit/etc/

logs/fluent-bit/image/VERSION

@@ -1 +1 @@
-v0.0.3
+v2.1.3

logs/fluent-bit/k8s-helm/http/Chart.yaml

@@ -1,14 +1,14 @@
 apiVersion: v2
 name: fluent-bit-http
 description: Fluent-Bit Chart with HTTP output plugin
-version: 0.1.0
-appVersion: 2.0.8
+version: 2.1.3
+appVersion: 2.1.3
 keywords:
   - Fluent-Bit
   - HTTP output plugin
 dependencies:
   - name: fluent-bit
-    version: "0.23.0"
+    version: "0.30.4"
     repository: https://fluent.github.io/helm-charts
 sources:
   - https://github.com/fluent/fluent-bit/

logs/fluent-bit/k8s-helm/http/values.yaml

@@ -3,7 +3,7 @@ fluent-bit:
 
   image:
     repository: coralogixrepo/coralogix-fluent-bit-multiarch
-    tag: v0.0.3
+    tag: v2.1.3
 
   serviceMonitor:
     enabled: true

logs/fluent-bit/k8s-manifest/README.md

@@ -28,19 +28,19 @@ Note: the configmap name is important and is being used by the daemonSet.
 
 change 'ENDPOINT' according to your logs endpoint from the table below.
 
-And apply it:
+And apply it to the monitoring namespace:
 
 ```bash
-kubectl apply -f fluentbit-env-cm.yaml
+kubectl apply -f fluentbit-env-cm.yaml -n monitoring
 ```
 
 Next apply the manifest files in this directory:
 
 ```bash
-kubectl apply -f https://raw.githubusercontent.com/coralogix/telemetry-shippers/master/logs/fluent-bit/k8s-manifest/fluentbit-cm.yaml
-kubectl apply -f https://raw.githubusercontent.com/coralogix/telemetry-shippers/master/logs/fluent-bit/k8s-manifest/fluentbit-rbac.yaml
-kubectl apply -f https://raw.githubusercontent.com/coralogix/telemetry-shippers/master/logs/fluent-bit/k8s-manifest/fluentbit-svc.yaml
-kubectl apply -f https://raw.githubusercontent.com/coralogix/telemetry-shippers/master/logs/fluent-bit/k8s-manifest/fluentbit-ds.yaml
+kubectl apply -f https://raw.githubusercontent.com/coralogix/telemetry-shippers/master/logs/fluent-bit/k8s-manifest/fluentbit-cm.yaml -n monitoring
+kubectl apply -f https://raw.githubusercontent.com/coralogix/telemetry-shippers/master/logs/fluent-bit/k8s-manifest/fluentbit-rbac.yaml -n monitoring
+kubectl apply -f https://raw.githubusercontent.com/coralogix/telemetry-shippers/master/logs/fluent-bit/k8s-manifest/fluentbit-svc.yaml -n monitoring
+kubectl apply -f https://raw.githubusercontent.com/coralogix/telemetry-shippers/master/logs/fluent-bit/k8s-manifest/fluentbit-ds.yaml -n monitoring
 ```
 
 The output should be :
@@ -58,7 +58,7 @@ service/fluent-bit created
 If you have prometheus-operator installed you can also install this service monitor resource:
 
 ```bash
-kubectl apply -f https://raw.githubusercontent.com/coralogix/telemetry-shippers/master/logs/fluent-bit/k8s-manifest/fluentbit-svc-monitor.yaml
+kubectl apply -f https://raw.githubusercontent.com/coralogix/telemetry-shippers/master/logs/fluent-bit/k8s-manifest/fluentbit-svc-monitor.yaml -n monitoring
 ```
 
 ## Modifying applicationName and subsystemName
@@ -119,11 +119,11 @@ removed for brevity...
 To remove all resources created with manifest files use these commands:
 
 ```bash
-kubectl delete -f https://raw.githubusercontent.com/coralogix/telemetry-shippers/master/logs/fluent-bit/k8s-manifest/fluentbit-cm.yaml
-kubectl delete -f https://raw.githubusercontent.com/coralogix/telemetry-shippers/master/logs/fluent-bit/k8s-manifest/fluentbit-rbac.yaml
-kubectl delete -f https://raw.githubusercontent.com/coralogix/telemetry-shippers/master/logs/fluent-bit/k8s-manifest/fluentbit-svc.yaml
-kubectl delete -f https://raw.githubusercontent.com/coralogix/telemetry-shippers/master/logs/fluent-bit/k8s-manifest/fluentbit-ds.yaml
-kubectl delete -f fluentbit-env-cm.yaml
+kubectl delete -f https://raw.githubusercontent.com/coralogix/telemetry-shippers/master/logs/fluent-bit/k8s-manifest/fluentbit-cm.yaml -n monitoring
+kubectl delete -f https://raw.githubusercontent.com/coralogix/telemetry-shippers/master/logs/fluent-bit/k8s-manifest/fluentbit-rbac.yaml -n monitoring
+kubectl delete -f https://raw.githubusercontent.com/coralogix/telemetry-shippers/master/logs/fluent-bit/k8s-manifest/fluentbit-svc.yaml -n monitoring
+kubectl delete -f https://raw.githubusercontent.com/coralogix/telemetry-shippers/master/logs/fluent-bit/k8s-manifest/fluentbit-ds.yaml -n monitoring
+kubectl delete -f fluentbit-env-cm.yaml -n monitoring
 ```
 
 The output should be :

logs/fluent-bit/k8s-manifest/fluentbit-cm.yaml

@@ -148,14 +148,24 @@ data:
   script.lua: |
       RETURN_WITHOUT_MODIFYING_TIMESTAMP = 2
       function addcrxmetadata(tag, timestamp, record)
-        new_record = record
-        new_record["applicationName"] = record.json.kubernetes.namespace_name
-        if new_record["applicationName"] == nil then
-        new_record["applicationName"] = "no-application" end
-        new_record["subsystemName"] = record.json.kubernetes.container_name
-        if new_record["subsystemName"] == nil then
-        new_record["subsystemName"] = "no-subsystem" end
-        local processed_fraction = string.format("%09d", timestamp['nsec'])
-        new_record["timestamp"] = string.format("%s%s", timestamp['sec'], string.sub(processed_fraction, 1, -4))
-        return RETURN_WITHOUT_MODIFYING_TIMESTAMP, timestamp, new_record
+          new_record = record
+
+          if record.json and record.json.kubernetes then
+              new_record["applicationName"] = record.json.kubernetes.namespace_name
+              if new_record["applicationName"] == nil then
+                  new_record["applicationName"] = "no-application"
+              end
+
+              new_record["subsystemName"] = record.json.kubernetes.container_name
+              if new_record["subsystemName"] == nil then
+                  new_record["subsystemName"] = "no-subsystem"
+              end
+          else
+              new_record["applicationName"] = "no-application"
+              new_record["subsystemName"] = "no-subsystem"
+          end
+
+          local processed_fraction = string.format("%09d", timestamp['nsec'])
+          new_record["timestamp"] = string.format("%s%s", timestamp['sec'], string.sub(processed_fraction, 1, -4))
+          return RETURN_WITHOUT_MODIFYING_TIMESTAMP, timestamp, new_record
       end

logs/fluent-bit/k8s-manifest/fluentbit-ds.yaml

@@ -21,7 +21,7 @@ spec:
       dnsPolicy: ClusterFirstWithHostNet
       containers:
         - name: fluent-bit
-          image: "coralogixrepo/coralogix-fluent-bit-multiarch:v0.0.2"
+          image: "coralogixrepo/coralogix-fluent-bit-multiarch:v2.1.3"
           imagePullPolicy: Always
           env:
             - name: HOSTNAME
@@ -34,6 +34,8 @@ spec:
                 name: coralogix-keys
             - configMapRef:
                 name: fluent-bit
+            - configMapRef:
+                name: fluent-bit-env
           ports:
             - name: http
               containerPort: 2020