Expose --conntrack-max-per-core kube-proxy flag

This commit exposes --conntrack-max-per-core kube-proxy flag in
kubernetes Helm chart and adds required plumbing to expose it to the
user using HCL.

It also adds sample usage to CI configuration and e2e tests to verify
that settings are properly applied.

Closes #1081

Signed-off-by: Mateusz Gozdek <mateusz@kinvolk.io>

assets/charts/control-plane/kubernetes/templates/kube-proxy.yaml

@@ -42,6 +42,9 @@ spec:
         - --proxy-mode=iptables
         - --metrics-bind-address=$(HOST_IP)
         - --healthz-bind-address=$(HOST_IP)
+        {{- if not (eq (int .Values.kubeProxy.conntrackMaxPerCore) 32768) }}
+        - --conntrack-max-per-core={{ .Values.kubeProxy.conntrackMaxPerCore }}
+        {{- end }}
         env:
           - name: NODE_NAME
             valueFrom:

assets/charts/control-plane/kubernetes/values.yaml

@@ -12,6 +12,7 @@ kubeProxy:
   image: k8s.gcr.io/kube-proxy:v1.19.3
   podCIDR: 10.2.0.0/16
   trustedCertsDir: /usr/share/ca-certificates
+  conntrackMaxPerCore: 32768
 kubeScheduler:
   image: k8s.gcr.io/kube-scheduler:v1.19.3
   controlPlaneReplicas: 1

assets/terraform-modules/aws/flatcar-linux/kubernetes/bootkube.tf

@@ -33,4 +33,6 @@ module "bootkube" {
   encrypt_pod_traffic  = var.encrypt_pod_traffic
 
   ignore_x509_cn_check = var.ignore_x509_cn_check
+
+  conntrack_max_per_core = var.conntrack_max_per_core
 }

assets/terraform-modules/aws/flatcar-linux/kubernetes/variables.tf

@@ -196,3 +196,8 @@ variable "ignore_x509_cn_check" {
   type        = bool
   default     = false
 }
+
+variable "conntrack_max_per_core" {
+  description = "--conntrack-max-per-core value for kube-proxy. Maximum number of NAT connections to track per CPU core (0 to leave the limit as-is and ignore conntrack-min)."
+  type        = number
+}

assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/bootkube.tf

@@ -26,4 +26,6 @@ module "bootkube" {
   encrypt_pod_traffic  = var.encrypt_pod_traffic
 
   ignore_x509_cn_check = var.ignore_x509_cn_check
+
+  conntrack_max_per_core = var.conntrack_max_per_core
 }

assets/terraform-modules/bare-metal/flatcar-linux/kubernetes/variables.tf

@@ -195,3 +195,8 @@ variable "ignore_x509_cn_check" {
   type        = bool
   default     = false
 }
+
+variable "conntrack_max_per_core" {
+  description = "--conntrack-max-per-core value for kube-proxy. Maximum number of NAT connections to track per CPU core (0 to leave the limit as-is and ignore conntrack-min)."
+  type        = number
+}

assets/terraform-modules/bootkube/assets.tf

@@ -82,6 +82,7 @@ resource "local_file" "kubernetes" {
     serviceaccount_key            = base64encode(tls_private_key.service-account.private_key_pem)
     etcd_endpoints                = var.etcd_endpoints
     enable_tls_bootstrap          = var.enable_tls_bootstrap
+    conntrack_max_per_core        = var.conntrack_max_per_core
   })
 }
 

assets/terraform-modules/bootkube/resources/charts/kubernetes.yaml

@@ -12,6 +12,7 @@ kubeProxy:
   image: ${kube_proxy_image}
   podCIDR: ${pod_cidr}
   trustedCertsDir: ${trusted_certs_dir}
+  conntrackMaxPerCore: ${conntrack_max_per_core}
 kubeScheduler:
   image: ${kube_scheduler_image}
   controlPlaneReplicas: ${control_plane_replicas}

assets/terraform-modules/bootkube/variables.tf

@@ -185,3 +185,8 @@ variable "encrypt_pod_traffic" {
   type        = bool
   default     = false
 }
+
+variable "conntrack_max_per_core" {
+  description = "--conntrack-max-per-core value for kube-proxy. Maximum number of NAT connections to track per CPU core (0 to leave the limit as-is and ignore conntrack-min)."
+  type        = number
+}

assets/terraform-modules/packet/flatcar-linux/kubernetes/bootkube.tf

@@ -48,4 +48,6 @@ module "bootkube" {
   encrypt_pod_traffic         = var.encrypt_pod_traffic
 
   ignore_x509_cn_check = var.ignore_x509_cn_check
+
+  conntrack_max_per_core = var.conntrack_max_per_core
 }

assets/terraform-modules/packet/flatcar-linux/kubernetes/variables.tf

@@ -213,3 +213,8 @@ variable "ignore_x509_cn_check" {
   type        = bool
   default     = false
 }
+
+variable "conntrack_max_per_core" {
+  description = "--conntrack-max-per-core value for kube-proxy. Maximum number of NAT connections to track per CPU core (0 to leave the limit as-is and ignore conntrack-min)."
+  type        = number
+}

ci/aws/aws-cluster.lokocfg.envsubst

@@ -42,8 +42,9 @@ EOF
     instance_type = "i3.large"
     spot_price    = "0.08"
     labels        = {
-      "testing.io" = "yes",
-      "roleofnode" = "testing",
+      "testing.io"          = "yes",
+      "roleofnode"          = "testing",
+      "conntrack-modified"  = "true",
     }
     tags = {
       "deployment" = "ci"
@@ -62,6 +63,24 @@ storage:
       group:
         id: 500
 EOF
+      ,
+      <<EOF
+storage:
+  files:
+    - path: /etc/modules-load.d/nf.conf
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          nf_conntrack
+    - path: /etc/sysctl.d/nf.conf
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          net.netfilter.nf_conntrack_max=50000
+EOF
+      ,
     ]
   }
 
@@ -72,8 +91,9 @@ EOF
     instance_type = "t2.small"
     spot_price    = "0.01"
     labels        = {
-      "testing.io" = "yes",
-      "roleofnode" = "testing",
+      "testing.io"          = "yes",
+      "roleofnode"          = "testing",
+      "conntrack-modified"  = "true",
     }
     taints        = {
       "nodeType" = "storage:NoSchedule"
@@ -99,13 +119,35 @@ storage:
       group:
         id: 500
 EOF
+      ,
+      <<EOF
+storage:
+  files:
+    - path: /etc/modules-load.d/nf.conf
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          nf_conntrack
+    - path: /etc/sysctl.d/nf.conf
+      filesystem: root
+      mode: 0644
+      contents:
+        inline: |
+          net.netfilter.nf_conntrack_max=50000
+EOF
+      ,
     ]
   }
 
   # Adds oidc flags to API server with default values.
   # Acts as a smoke test to check if API server is functional after addition
   # of extra flags.
   oidc {}
+
+  # Disable kube-proxy setting net.netfilter.nf_conntrack_max so we can
+  # set it per worker pool via CLC snippet.
+  conntrack_max_per_core = 0
 }
 
 component "metrics-server" {}

ci/baremetal/baremetal-cluster.lokocfg.envsubst

@@ -40,6 +40,8 @@ cluster "bare-metal" {
     "testing.io" = "yes",
     "roleofnode" = "testing",
   }
+
+  conntrack_max_per_core = 65000
 }
 
 component "inspektor-gadget" {}

ci/packet/packet-cluster.lokocfg.envsubst

@@ -62,6 +62,8 @@ EOF
   oidc {}
 
   ignore_x509_cn_check = true
+
+  conntrack_max_per_core = 65000
 }
 
 component "metrics-server" {}