[cilium] fix rbac and upgrade hubble v0.11.0 (#3) (kubernetes-sigs#9959)

* [cilium] fix rbac and upgrade hubble v0.11.0 (#3) * [cilium] fix rbac for LB bgp ipam * [cilium] Upgrade Hubble to v0.11.0 and add mTLS between Hubble UI and Hubble Relay * fix dns domain hubble for tls --------- Co-authored-by: Thuon Jeremy <d107869@olinfra1.infra.bdm.outscale.c1.dav.fr> * Fix blank line --------- Co-authored-by: Thuon Jeremy <d107869@olinfra1.infra.bdm.outscale.c1.dav.fr>
jeremythuon · Apr 10, 2023 · 4a03d13 · 4a03d13
1 parent fcb5e77
commit 4a03d13
Show file tree

Hide file tree

Showing 7 changed files with 66 additions and 28 deletions.
diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
@@ -1038,9 +1038,9 @@ cilium_hubble_relay_image_tag: "{{ cilium_version }}"
 cilium_hubble_certgen_image_repo: "{{ quay_image_repo }}/cilium/certgen"
 cilium_hubble_certgen_image_tag: "v0.1.8"
 cilium_hubble_ui_image_repo: "{{ quay_image_repo }}/cilium/hubble-ui"
-cilium_hubble_ui_image_tag: "v0.9.2"
+cilium_hubble_ui_image_tag: "v0.11.0"
 cilium_hubble_ui_backend_image_repo: "{{ quay_image_repo }}/cilium/hubble-ui-backend"
-cilium_hubble_ui_backend_image_tag: "v0.9.2"
+cilium_hubble_ui_backend_image_tag: "v0.11.0"
 cilium_hubble_envoy_image_repo: "{{ docker_image_repo }}/envoyproxy/envoy"
 cilium_hubble_envoy_image_tag: "v1.22.5"
 kube_ovn_container_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn"

diff --git a/roles/network_plugin/cilium/defaults/main.yml b/roles/network_plugin/cilium/defaults/main.yml
@@ -273,3 +273,20 @@ cilium_rolling_restart_wait_retries_delay_seconds: 10
 cilium_agent_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9962', '9090') }}"
 cilium_operator_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9963', '6942') }}"
 cilium_hubble_scrape_port: "{{ cilium_version | regex_replace('v') is version('1.12', '>=') | ternary('9965', '9091') }}"
+
+# Cilium certgen args for generate certificate for hubble mTLS
+cilium_certgen_args:
+  cilium-namespace: kube-system
+  ca-reuse-secret: true
+  ca-secret-name: hubble-ca-secret
+  ca-generate: true
+  ca-validity-duration: 94608000s
+  hubble-server-cert-generate: true
+  hubble-server-cert-common-name: '*.{{ cilium_cluster_name }}.hubble-grpc.cilium.io'
+  hubble-server-cert-validity-duration: 94608000s
+  hubble-server-cert-secret-name: hubble-server-certs
+  hubble-relay-client-cert-generate: true
+  hubble-relay-client-cert-common-name: '*.{{ cilium_cluster_name }}.hubble-grpc.cilium.io'
+  hubble-relay-client-cert-validity-duration: 94608000s
+  hubble-relay-client-cert-secret-name: hubble-relay-client-certs
+  hubble-relay-server-cert-generate: false
diff --git a/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2 b/roles/network_plugin/cilium/templates/cilium-operator/cr.yml.j2
@@ -54,6 +54,7 @@ rules:
   - services/status
   verbs:
   - update
+  - patch
 - apiGroups:
   - ""
   resources:
@@ -92,6 +93,8 @@ rules:
 {% endif %}
 {% if cilium_version | regex_replace('v') is version('1.12', '>=') %}
   - ciliumbgploadbalancerippools
+  - ciliumloadbalancerippools
+  - ciliumloadbalancerippools/status
   - ciliumbgppeeringpolicies
   - ciliumenvoyconfigs
 {% endif %}

diff --git a/roles/network_plugin/cilium/templates/hubble/cronjob.yml.j2 b/roles/network_plugin/cilium/templates/hubble/cronjob.yml.j2
@@ -29,19 +29,10 @@ spec:
               # line args instead of via config map. This allows users to inspect
               # the values used in past runs by inspecting the completed pod.
               args:
-                - "--cilium-namespace=kube-system"
-                - "--ca-reuse-secret=true"
-                - "--ca-secret-name=hubble-ca-secret"
-                - "--ca-generate=true"
-                - "--ca-validity-duration=94608000s"
-                - "--hubble-server-cert-generate=true"
-                - "--hubble-server-cert-common-name=*.{{ cilium_cluster_name }}.hubble-grpc.cilium.io"
-                - "--hubble-server-cert-validity-duration=94608000s"
-                - "--hubble-server-cert-secret-name=hubble-server-certs"
-                - "--hubble-relay-client-cert-generate=true"
-                - "--hubble-relay-client-cert-validity-duration=94608000s"
-                - "--hubble-relay-client-cert-secret-name=hubble-relay-client-certs"
-                - "--hubble-relay-server-cert-generate=false"
+              {% for key, value in cilium_certgen_args.items() -%}
+                - "--{{ key }}={{ value }}"
+              {% endfor %}
+
           hostNetwork: true
           restartPolicy: OnFailure
       ttlSecondsAfterFinished: 1800
diff --git a/roles/network_plugin/cilium/templates/hubble/deploy.yml.j2 b/roles/network_plugin/cilium/templates/hubble/deploy.yml.j2
@@ -138,8 +138,28 @@ spec:
           env:
             - name: EVENTS_SERVER_PORT
               value: "8090"
+            {% if cilium_hubble_tls_generate -%}
+            - name: TLS_TO_RELAY_ENABLED
+              value: "true"
+            - name: FLOWS_API_ADDR
+              value: "hubble-relay:443"
+            - name: TLS_RELAY_SERVER_NAME
+              value: ui.{{ cilium_cluster_name }}.hubble-grpc.cilium.io
+            - name: TLS_RELAY_CA_CERT_FILES
+              value: /var/lib/hubble-ui/certs/hubble-server-ca.crt
+            - name: TLS_RELAY_CLIENT_CERT_FILE
+              value: /var/lib/hubble-ui/certs/client.crt
+            - name: TLS_RELAY_CLIENT_KEY_FILE
+              value: /var/lib/hubble-ui/certs/client.key
+            {% else -%}
             - name: FLOWS_API_ADDR
               value: "hubble-relay:80"
+            {% endif %}
+
+          volumeMounts:
+            - name: tls
+              mountPath: /var/lib/hubble-ui/certs
+              readOnly: true
           ports:
             - containerPort: 8090
               name: grpc
@@ -150,5 +170,17 @@ spec:
             defaultMode: 420
             name: hubble-ui-nginx
           name: hubble-ui-nginx-conf
+        - projected:
+            sources:
+            - secret:
+                name: hubble-relay-client-certs
+                items:
+                  - key: ca.crt
+                    path: hubble-server-ca.crt
+                  - key: tls.crt
+                    path: client.crt
+                  - key: tls.key
+                    path: client.key
+          name: tls
         - emptyDir: {}
           name: tmp-dir
diff --git a/roles/network_plugin/cilium/templates/hubble/job.yml.j2 b/roles/network_plugin/cilium/templates/hubble/job.yml.j2
@@ -25,19 +25,10 @@ spec:
           # line args instead of via config map. This allows users to inspect
           # the values used in past runs by inspecting the completed pod.
           args:
-            - "--cilium-namespace=kube-system"
-            - "--ca-reuse-secret=true"
-            - "--ca-secret-name=hubble-ca-secret"
-            - "--ca-generate=true"
-            - "--ca-validity-duration=94608000s"
-            - "--hubble-server-cert-generate=true"
-            - "--hubble-server-cert-common-name=*.{{ cilium_cluster_name }}.hubble-grpc.cilium.io"
-            - "--hubble-server-cert-validity-duration=94608000s"
-            - "--hubble-server-cert-secret-name=hubble-server-certs"
-            - "--hubble-relay-client-cert-generate=true"
-            - "--hubble-relay-client-cert-validity-duration=94608000s"
-            - "--hubble-relay-client-cert-secret-name=hubble-relay-client-certs"
-            - "--hubble-relay-server-cert-generate=false"
+          {% for key, value in cilium_certgen_args.items() -%}
+            - "--{{ key }}={{ value }}"
+          {% endfor %}
+
       hostNetwork: true
       restartPolicy: OnFailure
   ttlSecondsAfterFinished: 1800
diff --git a/roles/network_plugin/cilium/templates/hubble/service.yml.j2 b/roles/network_plugin/cilium/templates/hubble/service.yml.j2
@@ -58,7 +58,11 @@ spec:
     k8s-app: hubble-relay
   ports:
   - protocol: TCP
+    {% if cilium_hubble_tls_generate -%}
+    port: 443
+    {% else -%}
     port: 80
+    {% endif -%}
     targetPort: 4245
 ---
 # Source: cilium/templates/hubble-ui-service.yaml