update service from loadbalance to nodeport

alipay · Feb 29, 2024 · 35802c4 · 35802c4
2 parents 718f233 + 6a67c2f
commit 35802c4
Show file tree

Hide file tree

Showing 4 changed files with 89 additions and 18 deletions.
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
@@ -53,25 +53,38 @@ jobs:
           --set jaegerType=NodePort
         # waiting for lunettes ready
         set +e
+        all_pods=$(kubectl -n lunettes get pods -o jsonpath='{.items[*].metadata.name}')
+        echo "pods are: ${all_pods}"
         for ((i=0; i<120; i++))
         do
-          status=$(curl -s -XGET "http://localhost:9094/_cluster/health?pretty" | awk -F'"' '/"status"/{print $4}')
-          echo "status is $status"
-          if [ -n "$status" ] && [ "$status" != "red" ]; then
-              echo "Elasticsearch服务已准备就绪，状态为: $status"
+          all_pods_running=true
+          for pod in ${all_pods}
+          do
+            status=$(kubectl -n lunettes get pod ${pod} --output=jsonpath="{.status.phase}" )
+            if [ "${status}" != "Running" ]
+            then
+              all_pods_running=false
               break
-          else
-              echo "Elasticsearch服务未准备就绪，当前状态为: $status"
+            fi
+          done
+          if [ ${all_pods_running} = "true" ]
+          then
+            echo "all pods are running"
+            break
           fi
           output=$(kubectl -n lunettes get pods -owide)
           echo "$output"
-          lunettes_pod_name=$(echo "$output" | awk '$1 ~ /^lunettes-/{print $1}')
-          echo -e "################# log lunettes ......#################\n"
-          kubectl -n lunettes logs $lunettes_pod_name | head -n 100
-          es_pod_name=$(echo "$output" | awk '$1 ~ /^es-single-/{print $1}')
+          lunettes_pod_name=$(kubectl -n lunettes get pods | awk '$1 ~ /^lunettes-/{print $1}')
+          echo -e "################# log lunettes init ......#################\n"
+          kubectl -n lunettes logs ${lunettes_pod_name} -c init |head -n 100
+          grafanadi_pod_name=$(kubectl -n lunettes get pods | awk '$1 ~ /^grafanadi-/{print $1}')
+          echo -e "################# log grafanadi init ......#################\n"
+          kubectl -n lunettes logs ${grafanadi_pod_name} -c init |head -n 100
+          es_pod_name=$(kubectl -n lunettes get pods | awk '$1 ~ /^es-single-/{print $1}')
           echo -e "################# log es-single ...... #################\n"
-          kubectl -n lunettes logs $es_pod_name | head -n 100
-          sleep 1m
+          kubectl -n lunettes logs ${es_pod_name} | head -n 100
+          echo "rechecking the pod status after 30s..."
+          sleep 30
         done
         set -e
     - name: Create test pod

diff --git a/deploy/helm/lunettes/templates/grafanadi/grafanadi-deploy.yaml b/deploy/helm/lunettes/templates/grafanadi/grafanadi-deploy.yaml
@@ -18,11 +18,41 @@ spec:
         fsGroup: 472
         supplementalGroups:
           - 0
+      initContainers:
+      - command:
+        - bash
+        - -c
+        - |
+          echo "Waiting for Elasticsearch cluster to be ready..."
+          wait_time=1
+          # check es cluster health status
+          while true;
+          do
+            status=`curl -s -XGET --connect-timeout 3 --max-time 5 \
+            "http://es-cluster-svc.{{ .Values.namespace }}:9200/_cluster/health?pretty" \
+            | awk -F'"' '/"status"/{print $4}'`
+            if [ -n "${status}" ] && [ "${status}" != "red" ];
+            then
+              echo "Elasticsearch is healthy, status is: ${status}"
+              break
+            else
+              echo "Elasticsearch is unhealthy, status is: ${status}"
+              wait_time=$((wait_time+wait_time))
+              echo "Rechecking the status after ${wait_time}s..."
+              sleep ${wait_time}
+            fi
+          done
+        image: {{ .Values.lunettesImage }}
+        imagePullPolicy: {{ .Values.imagePullPolicy }}
+        name: init
       containers:
       - command:
         - ./grafanadi
         - --config-file=/app/config-file.yaml
         - --enable-kube-client={{ .Values.grafanadiInitKubeClient }}
+        - --log_dir={{ .Values.grafanadiLogsPath }}
+        - --logtostderr=false
+        - --v=6
         name: grafanadi
         image: {{ .Values.lunettesImage }}
         imagePullPolicy: {{ .Values.imagePullPolicy }}
@@ -33,6 +63,8 @@ spec:
         resources:
 {{ toYaml .Values.grafanadiResources | indent 10 }}
         volumeMounts:
+        - mountPath: {{ .Values.grafanadiLogsPath }}
+          name: logs
         - mountPath: /var/grafana
           name: grafana-pv
         - mountPath: /app/config-file.yaml
@@ -43,6 +75,10 @@ spec:
       - name: grafana-pv
         hostPath:
           path: /data/volume/grafana
+      - name: logs
+        hostPath:
+          path: {{ .Values.grafanadiLogsHostPath }}
+          type: DirectoryOrCreate
       - configMap:
           defaultMode: 420
           name: grafanadi-cm

diff --git a/deploy/helm/lunettes/templates/lunettes/lunettes-deploy.yaml b/deploy/helm/lunettes/templates/lunettes/lunettes-deploy.yaml
@@ -22,7 +22,27 @@ spec:
       - command:
         - bash
         - -c
-        - chown -R 500:500 /logs
+        - |
+          chown -R 500:500 {{ .Values.lunettesLogsPath }}
+          echo "Waiting for Elasticsearch cluster to be ready..."
+          wait_time=1
+          # check es cluster health status
+          while true;
+          do
+            status=`curl -s -XGET --connect-timeout 3 --max-time 5 \
+            "http://es-cluster-svc.{{ .Values.namespace }}:9200/_cluster/health?pretty" \
+            | awk -F'"' '/"status"/{print $4}'`
+            if [ -n "${status}" ] && [ "${status}" != "red" ];
+            then
+              echo "Elasticsearch is healthy, status is: ${status}"
+              break
+            else
+              echo "Elasticsearch is unhealthy, status is: ${status}"
+              wait_time=$((wait_time+wait_time))
+              echo "Rechecking the status after ${wait_time}s..."
+              sleep ${wait_time}
+            fi
+          done
         image: {{ .Values.lunettesImage }}
         imagePullPolicy: {{ .Values.imagePullPolicy }}
         name: init
@@ -32,7 +52,7 @@ spec:
         terminationMessagePath: /dev/termination-log
         terminationMessagePolicy: File
         volumeMounts:
-        - mountPath: /logs
+        - mountPath: {{ .Values.lunettesLogsPath }}
           name: logs
       containers:
       - name: {{ template "lunettes.fullname" . }}
@@ -46,8 +66,8 @@ spec:
         - --otlp-collector=jaeger-collector.{{ .Values.namespace }}:4317
         - --v=6
         - --apiserver-enabled={{ .Values.apiserverEnabled }}
-        - --log_dir=/logs
-        - --logtostderr=true
+        - --log_dir={{ .Values.lunettesLogsPath }}
+        - --logtostderr=false
         - --es-buffer-duration=10s
         - --es-fetch-interval=1s
         - --es-endpoint=http://es-cluster-svc.{{ .Values.namespace }}:9200
@@ -83,7 +103,7 @@ spec:
           runAsNonRoot: true
           runAsUser: 500          
         volumeMounts:
-        - mountPath: /logs
+        - mountPath: {{ .Values.lunettesLogsPath }}
           name: logs
         resources: 
 {{ toYaml .Values.lunettesResources | indent 10 }}
@@ -101,6 +121,6 @@ spec:
                 operator: Exists
       volumes:
       - hostPath:
-          path: /home/var/logs/lunettes/
+          path: {{ .Values.lunettesLogsHostPath }}
           type: DirectoryOrCreate
         name: logs
diff --git a/deploy/helm/lunettes/values.yaml b/deploy/helm/lunettes/values.yaml
@@ -42,6 +42,8 @@ lunettesResources:
 grafanadiInitKubeClient: false
 grafanadiType: NodePort
 grafanadiNodePort: 30280
+grafanadiLogsPath: /logs
+grafanadiLogsHostPath: /home/var/logs/grafanadi
 grafanadiResources:
   limits:
     cpu: 250m