Update MetalLB and switch to CRD notation. (#9120)

Signed-off-by: Jeroen Rijken <jeroen.rijken@xs4all.nl>
kubernetes-sigs · Apr 14, 2023 · 709ae1d · 709ae1d
1 parent 73ce6ae
commit 709ae1d
Show file tree

Hide file tree

Showing 10 changed files with 2,171 additions and 327 deletions.
diff --git a/README.md b/README.md
@@ -177,7 +177,7 @@ Note: Upstart/SysV init based OS types are not supported.
   - [krew](https://github.com/kubernetes-sigs/krew) v0.4.3
   - [argocd](https://argoproj.github.io/) v2.6.7
   - [helm](https://helm.sh/) v3.11.2
-  - [metallb](https://metallb.universe.tf/)  v0.12.1
+  - [metallb](https://metallb.universe.tf/)  v0.13.9
   - [registry](https://github.com/distribution/distribution) v2.8.1
 - Storage Plugin
   - [cephfs-provisioner](https://github.com/kubernetes-incubator/external-storage) v2.1.0-k8s1.11

diff --git a/docs/metallb.md b/docs/metallb.md
@@ -14,58 +14,121 @@ kube_proxy_strict_arp: true
 
 ## Install
 
-You have to explicitly enable the MetalLB extension and set an IP address range from which to allocate LoadBalancer IPs.
+You have to explicitly enable the MetalLB extension.
 
 ```yaml
 metallb_enabled: true
 metallb_speaker_enabled: true
-metallb_avoid_buggy_ips: true
-metallb_ip_range:
-  - 10.5.0.0/16
 ```
 
 By default only the MetalLB BGP speaker is allowed to run on control plane nodes. If you have a single node cluster or a cluster where control plane are also worker nodes you may need to enable tolerations for the MetalLB controller:
 
 ```yaml
-metallb_controller_tolerations:
-  - key: "node-role.kubernetes.io/master"
-    operator: "Equal"
-    value: ""
-    effect: "NoSchedule"
-  - key: "node-role.kubernetes.io/control-plane"
-    operator: "Equal"
-    value: ""
-    effect: "NoSchedule"
+metallb_config:
+  controller:
+    tolerations:
+      - key: "node-role.kubernetes.io/master"
+        operator: "Equal"
+        value: ""
+        effect: "NoSchedule"
+      - key: "node-role.kubernetes.io/control-plane"
+        operator: "Equal"
+        value: ""
+        effect: "NoSchedule"
 ```
 
-## BGP Mode
+## Pools
 
-When operating in BGP Mode MetalLB needs to have defined upstream peers:
+First you need to specify all of the pools you are going to use:
 
 ```yaml
-metallb_protocol: bgp
-metallb_ip_range:
-  - 10.5.0.0/16
-metallb_peers:
-  - peer_address: 192.0.2.1
-    peer_asn: 64512
-    my_asn: 4200000000
-  - peer_address: 192.0.2.2
-    peer_asn: 64513
-    my_asn: 4200000000
+metallb_config:
+
+  address_pools:
+
+    primary:
+      ip_range:
+        - 192.0.1.0-192.0.1.254
+      auto_assign: true
+
+    pool1:
+      ip_range:
+        - 192.0.2.1-192.0.2.1
+      auto_assign: false # When set to false, you need to explicitly set the loadBalancerIP in the service!
+
+    pool2:
+      ip_range:
+        - 192.0.2.2-192.0.2.2
+      auto_assign: false
 ```
 
-Some upstream BGP peers may require password authentication:
+## Layer2 Mode
+
+Pools that need to be configured in layer2 mode, need to be specified in a list:
 
 ```yaml
-metallb_protocol: bgp
-metallb_ip_range:
-  - 10.5.0.0/16
-metallb_peers:
-  - peer_address: 192.0.2.1
-    peer_asn: 64512
-    my_asn: 4200000000
-    password: "changeme"
+metallb_config:
+
+  layer2:
+    - primary
+```
+
+## BGP Mode
+
+When operating in BGP Mode MetalLB needs to have defined upstream peers and link the pool(s) specified above to the correct peer:
+
+```yaml
+metallb_config:
+
+  layer3:
+    defaults:
+
+      peer_port: 179 # The TCP port to talk to. Defaults to 179, you shouldn't need to set this in production.
+      hold_time: 120s # Requested BGP hold time, per RFC4271.
+
+    communities:
+      vpn-only: "1234:1"
+      NO_ADVERTISE: "65535:65282"
+
+    metallb_peers:
+
+        peer1:
+          peer_address: 192.0.2.1
+          peer_asn: 64512
+          my_asn: 4200000000
+          communities:
+            - vpn-only
+          address_pool:
+            - pool1
+
+          # (optional) The source IP address to use when establishing the BGP session. In most cases the source-address field should only be used with per-node peers, i.e. peers with node selectors which select only one node. CURRENTLY NOT SUPPORTED
+          source_address: 192.0.2.2
+
+          # (optional) The router ID to use when connecting to this peer. Defaults to the node IP address.
+          # Generally only useful when you need to peer with another BGP router running on the same machine as MetalLB.
+          router_id: 1.2.3.4
+
+          # (optional) Password for TCPMD5 authenticated BGP sessions offered by some peers.
+          password: "changeme"
+
+        peer2:
+          peer_address: 192.0.2.2
+          peer_asn: 64513
+          my_asn: 4200000000
+          communities:
+            - NO_ADVERTISE
+          address_pool:
+            - pool2
+
+          # (optional) The source IP address to use when establishing the BGP session. In most cases the source-address field should only be used with per-node peers, i.e. peers with node selectors which select only one node. CURRENTLY NOT SUPPORTED
+          source_address: 192.0.2.1
+
+          # (optional) The router ID to use when connecting to this peer. Defaults to the node IP address.
+          # Generally only useful when you need to peer with another BGP router running on the same machine as MetalLB.
+          router_id: 1.2.3.5
+
+          # (optional) Password for TCPMD5 authenticated BGP sessions offered by some peers.
+          password: "changeme"
 ```
 
 When using calico >= 3.18 you can replace MetalLB speaker by calico Service LoadBalancer IP advertisement.
@@ -75,30 +138,61 @@ In this scenario you should disable the MetalLB speaker and configure the `calic
 ```yaml
 metallb_speaker_enabled: false
 metallb_avoid_buggy_ips: true
-metallb_ip_range:
-  - 10.5.0.0/16
-calico_advertise_service_loadbalancer_ips: "{{ metallb_ip_range }}"
+metallb_config:
+  address_pools:
+    primary:
+      ip_range:
+        - 10.5.0.0/16
+      auto_assign: true
+  layer2:
+    - primary
+calico_advertise_service_loadbalancer_ips: "{{ metallb_config.address_pools.primary.ip_range }}"
 ```
 
-If you have additional loadbalancer IP pool in `metallb_additional_address_pools` , ensure to add them to the list.
+If you have additional loadbalancer IP pool in `metallb_config.address_pools` , ensure to add them to the list.
 
 ```yaml
 metallb_speaker_enabled: false
-metallb_ip_range:
-  - 10.5.0.0/16
-metallb_additional_address_pools:
-  kube_service_pool_1:
-    ip_range:
-      - 10.6.0.0/16
-    protocol: "bgp"
-    auto_assign: false
-    avoid_buggy_ips: true
-  kube_service_pool_2:
-    ip_range:
-      - 10.10.0.0/16
-    protocol: "bgp"
-    auto_assign: false
-    avoid_buggy_ips: true
+metallb_config:
+  address_pools:
+    primary:
+      ip_range:
+        - 10.5.0.0/16
+      auto_assign: true
+    pool1:
+      ip_range:
+        - 10.6.0.0/16
+      auto_assign: true
+    pool2:
+      ip_range:
+        - 10.10.0.0/16
+      auto_assign: true
+  layer2:
+    - primary
+  layer3:
+    defaults:
+      peer_port: 179
+      hold_time: 120s
+    communities:
+      vpn-only: "1234:1"
+      NO_ADVERTISE: "65535:65282"
+    metallb_peers:
+        peer1:
+          peer_address: 10.6.0.1
+          peer_asn: 64512
+          my_asn: 4200000000
+          communities:
+            - vpn-only
+          address_pool:
+            - pool1
+        peer2:
+          peer_address: 10.10.0.1
+          peer_asn: 64513
+          my_asn: 4200000000
+          communities:
+            - NO_ADVERTISE
+          address_pool:
+            - pool2
 calico_advertise_service_loadbalancer_ips:
   - 10.5.0.0/16
   - 10.6.0.0/16

diff --git a/inventory/sample/group_vars/k8s_cluster/addons.yml b/inventory/sample/group_vars/k8s_cluster/addons.yml
@@ -170,11 +170,6 @@ cert_manager_enabled: false
 # MetalLB deployment
 metallb_enabled: false
 metallb_speaker_enabled: "{{ metallb_enabled }}"
-# metallb_ip_range:
-#   - "10.5.0.50-10.5.0.99"
-# metallb_pool_name: "loadbalanced"
-# metallb_auto_assign: true
-# metallb_avoid_buggy_ips: false
 # metallb_speaker_nodeselector:
 #   kubernetes.io/os: "linux"
 # metallb_controller_nodeselector:
@@ -197,25 +192,50 @@ metallb_speaker_enabled: "{{ metallb_enabled }}"
 #     operator: "Equal"
 #     value: ""
 #     effect: "NoSchedule"
-# metallb_version: v0.12.1
+# metallb_version: v0.13.9
 # metallb_protocol: "layer2"
 # metallb_port: "7472"
 # metallb_memberlist_port: "7946"
-# metallb_additional_address_pools:
-#   kube_service_pool:
-#     ip_range:
-#       - "10.5.1.50-10.5.1.99"
-#     protocol: "layer2"
-#     auto_assign: false
-#     avoid_buggy_ips: false
-# metallb_protocol: "bgp"
-# metallb_peers:
-#   - peer_address: 192.0.2.1
-#     peer_asn: 64512
-#     my_asn: 4200000000
-#   - peer_address: 192.0.2.2
-#     peer_asn: 64513
-#     my_asn: 4200000000
+# metallb_config:
+#   address_pools:
+#     primary:
+#       ip_range:
+#         - 10.5.0.0/16
+#       auto_assign: true
+#     pool1:
+#       ip_range:
+#         - 10.6.0.0/16
+#       auto_assign: true
+#     pool2:
+#       ip_range:
+#         - 10.10.0.0/16
+#       auto_assign: true
+#   layer2:
+#     - primary
+#   layer3:
+#     defaults:
+#       peer_port: 179
+#       hold_time: 120s
+#     communities:
+#       vpn-only: "1234:1"
+#       NO_ADVERTISE: "65535:65282"
+#     metallb_peers:
+#         peer1:
+#           peer_address: 10.6.0.1
+#           peer_asn: 64512
+#           my_asn: 4200000000
+#           communities:
+#             - vpn-only
+#           address_pool:
+#             - pool1
+#         peer2:
+#           peer_address: 10.10.0.1
+#           peer_asn: 64513
+#           my_asn: 4200000000
+#           communities:
+#             - NO_ADVERTISE
+#           address_pool:
+#             - pool2
 
 argocd_enabled: false
 # argocd_version: v2.6.7

diff --git a/roles/download/defaults/main.yml b/roles/download/defaults/main.yml
@@ -1166,7 +1166,7 @@ dashboard_metrics_scraper_tag: "v1.0.8"
 
 metallb_speaker_image_repo: "{{ quay_image_repo }}/metallb/speaker"
 metallb_controller_image_repo: "{{ quay_image_repo }}/metallb/controller"
-metallb_version: v0.12.1
+metallb_version: v0.13.9
 
 downloads:
   netcheck_server:

diff --git a/roles/kubernetes-apps/metallb/tasks/main.yml b/roles/kubernetes-apps/metallb/tasks/main.yml
@@ -15,8 +15,8 @@
   fail:
     msg: "metallb_peers is mandatory when metallb_protocol is bgp and metallb_speaker_enabled"
   when:
-    - metallb_protocol == 'bgp' and metallb_speaker_enabled
-    - metallb_peers is not defined or not metallb_peers
+    - metallb_config.layer3 is defined and metallb_speaker_enabled
+    - metallb_config.metallb_peers is not defined or not metallb_config.metallb_peers
 
 - name: Kubernetes Apps | Check that the deprecated 'matallb_auto_assign' variable is not used anymore
   fail:
@@ -45,11 +45,29 @@
     src: "{{ item }}.j2"
     dest: "{{ kube_config_dir }}/{{ item }}"
     mode: 0644
-  with_items: ["metallb.yml", "metallb-config.yml"]
+  with_items: ["metallb.yml", "metallb-config.yml", "pools.yaml", "layer2.yaml", "layer3.yaml"]
   register: "rendering"
   when:
     - "inventory_hostname == groups['kube_control_plane'][0]"
 
+- name: Kubernetes Apps | Create MetalLB resources and replace existing
+  k8s:
+    definition: "{{ lookup('template', 'metallb.yaml') }}"
+
+- name: Kubernetes Apps | Wait for MetalLB controller to be running
+  k8s_info:
+    kind: Deployment
+    namespace: metallb-system
+    name: controller
+    wait: True
+    wait_sleep: 10
+    wait_timeout: 360
+    wait_condition:
+      status: "True"
+      type: Available
+  register: result
+  until: result is not failed
+
 - name: Kubernetes Apps | Install and configure MetalLB
   kube:
     name: "MetalLB"
@@ -60,3 +78,10 @@
   with_items: "{{ rendering.results }}"
   when:
     - "inventory_hostname == groups['kube_control_plane'][0]"
+
+- name: Kubernetes Apps | Delete MetalLB ConfigMap
+  k8s:
+    name: config
+    kind: ConfigMap
+    namespace: metallb-system
+    state: absent