diff --git a/packaging/flavorgen/flavors/kubevip/files.go b/packaging/flavorgen/flavors/kubevip/files.go index 934d21eaba..089058a85f 100644 --- a/packaging/flavorgen/flavors/kubevip/files.go +++ b/packaging/flavorgen/flavors/kubevip/files.go @@ -18,15 +18,24 @@ package kubevip import ( _ "embed" + "fmt" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1" + "sigs.k8s.io/yaml" ) var ( - // This two files are part of the workaround for https://github.com/kube-vip/kube-vip/issues/684 + // This file is part of the workaround for https://github.com/kube-vip/kube-vip/issues/684 //go:embed kube-vip-prepare.sh kubeVipPrepare string + + // kubeVipPodRaw yaml is generated via: + // docker run --network host --rm ghcr.io/kube-vip/kube-vip:${TAG} manifest pod --controlplane --address '${CONTROL_PLANE_ENDPOINT_IP}' --interface '${VIP_NETWORK_INTERFACE:=""}' --arp --leaderElection --leaseDuration 15 --leaseRenewDuration 10 --leaseRetry 2 --services --servicesElection > packaging/flavorgen/flavors/kubevip/kube-vip.yaml + //go:embed kube-vip.yaml + kubeVipPodRaw string ) func newKubeVIPFiles() []bootstrapv1.File { @@ -34,7 +43,7 @@ func newKubeVIPFiles() []bootstrapv1.File { { Owner: "root:root", Path: "/etc/kubernetes/manifests/kube-vip.yaml", - Content: kubeVIPPod(), + Content: kubeVIPPodYAML(), Permissions: "0644", }, // This file is part of the workaround for https://github.com/kube-vip/kube-vip/issues/692 @@ -53,3 +62,46 @@ func newKubeVIPFiles() []bootstrapv1.File { }, } } + +func kubeVIPPodYAML() string { + pod := &corev1.Pod{} + + if err := yaml.Unmarshal([]byte(kubeVipPodRaw), pod); err != nil { + panic(err) + } + + if len(pod.Spec.Containers) != 1 { + panic(fmt.Sprintf("Expected the kube-vip static pod manifest to have one container but got %d", len(pod.Spec.Containers))) + } + + // Set IfNotPresent to prevent unnecessary image pulls + pod.Spec.Containers[0].ImagePullPolicy = corev1.PullIfNotPresent + + // Apply workaround for https://github.com/kube-vip/kube-vip/issues/692 + // which is not using HostAliases, but a prebuilt /etc/hosts file instead. + pod.Spec.HostAliases = nil + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, + corev1.VolumeMount{ + Name: "etchosts", + MountPath: "/etc/hosts", + }, + ) + pod.Spec.Volumes = append(pod.Spec.Volumes, + corev1.Volume{ + Name: "etchosts", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/etc/kube-vip.hosts", + Type: ptr.To(corev1.HostPathFile), + }, + }, + }, + ) + + out, err := yaml.Marshal(pod) + if err != nil { + panic(err) + } + + return string(out) +} diff --git a/packaging/flavorgen/flavors/kubevip/kube-vip.yaml b/packaging/flavorgen/flavors/kubevip/kube-vip.yaml new file mode 100644 index 0000000000..d18d3d7ace --- /dev/null +++ b/packaging/flavorgen/flavors/kubevip/kube-vip.yaml @@ -0,0 +1,68 @@ +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: null + name: kube-vip + namespace: kube-system +spec: + containers: + - args: + - manager + env: + - name: vip_arp + value: "true" + - name: port + value: "6443" + - name: vip_interface + value: ${VIP_NETWORK_INTERFACE:=""} + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: svc_election + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leasename + value: plndr-cp-lock + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + - name: address + value: ${CONTROL_PLANE_ENDPOINT_IP} + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.6.4 + imagePullPolicy: Always + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + ip: 127.0.0.1 + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + name: kubeconfig +status: {} + diff --git a/packaging/flavorgen/flavors/kubevip/kubevip.go b/packaging/flavorgen/flavors/kubevip/kubevip.go index 6cf7982c05..24fcc782bd 100644 --- a/packaging/flavorgen/flavors/kubevip/kubevip.go +++ b/packaging/flavorgen/flavors/kubevip/kubevip.go @@ -18,162 +18,16 @@ limitations under the License. package kubevip import ( - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1" - "sigs.k8s.io/yaml" - - "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/env" - "sigs.k8s.io/cluster-api-provider-vsphere/packaging/flavorgen/flavors/util" -) - -var ( - hostPathTypeFile = corev1.HostPathFile - - kubeVipPodSpec = &corev1.Pod{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: util.TypeToKind(&corev1.Pod{}), - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "kube-vip", - Namespace: "kube-system", - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "kube-vip", - Image: "ghcr.io/kube-vip/kube-vip:v0.6.3", - ImagePullPolicy: corev1.PullIfNotPresent, - Args: []string{ - "manager", - }, - Env: []corev1.EnvVar{ - { - // Enables kube-vip control-plane functionality - Name: "cp_enable", - Value: "true", - }, - { - // Interface that the vip should bind to - Name: "vip_interface", - Value: env.VipNetworkInterfaceVar, - }, - { - // VIP IP address - // 'vip_address' was replaced by 'address' - Name: "address", - Value: env.ControlPlaneEndpointVar, - }, - { - // VIP TCP port - Name: "port", - Value: "6443", - }, - { - // Enables ARP brodcasts from Leader (requires L2 connectivity) - Name: "vip_arp", - Value: "true", - }, - { - // Kubernetes algorithm to be used. - Name: "vip_leaderelection", - Value: "true", - }, - { - // Seconds a lease is held for - Name: "vip_leaseduration", - Value: "15", - }, - { - // Seconds a leader can attempt to renew the lease - Name: "vip_renewdeadline", - Value: "10", - }, - { - // Number of times the leader will hold the lease for - Name: "vip_retryperiod", - Value: "2", - }, - { - // Enables kube-vip to watch Services of type LoadBalancer - Name: "svc_enable", - Value: "true", - }, - { - // Enables a leadership Election for each Service, allowing them to be distributed - Name: "svc_election", - Value: "true", - }, - }, - SecurityContext: &corev1.SecurityContext{ - Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{ - "NET_ADMIN", - "NET_RAW", - }, - }, - }, - VolumeMounts: []corev1.VolumeMount{ - { - MountPath: "/etc/kubernetes/admin.conf", - Name: "kubeconfig", - }, - // This mount is part of the workaround for https://github.com/kube-vip/kube-vip/issues/692 - { - MountPath: "/etc/hosts", - Name: "etchosts", - }, - }, - }, - }, - HostNetwork: true, - Volumes: []corev1.Volume{ - { - Name: "kubeconfig", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/etc/kubernetes/admin.conf", - Type: &hostPathTypeFile, - }, - }, - }, - // This mount is part of the workaround for https://github.com/kube-vip/kube-vip/issues/692 - { - Name: "etchosts", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: "/etc/kube-vip.hosts", - Type: &hostPathTypeFile, - }, - }, - }, - }, - }, - } ) +// PatchControlPlane adds kube-vip to a KubeadmControlPlane object. func PatchControlPlane(cp *controlplanev1.KubeadmControlPlane) { cp.Spec.KubeadmConfigSpec.Files = append(cp.Spec.KubeadmConfigSpec.Files, newKubeVIPFiles()...) - // This two commands are part of the workaround for https://github.com/kube-vip/kube-vip/issues/684 + // This commands is part of the workaround for https://github.com/kube-vip/kube-vip/issues/684 cp.Spec.KubeadmConfigSpec.PreKubeadmCommands = append( cp.Spec.KubeadmConfigSpec.PreKubeadmCommands, "/etc/kube-vip-prepare.sh", ) } - -// kubeVIPPodYaml converts the KubeVip pod spec to a `printable` yaml -// this is needed for the file contents of KubeadmConfig. -func kubeVIPPodYaml() string { - podYaml := util.GenerateObjectYAML(kubeVipPodSpec, []util.Replacement{}) - return podYaml -} - -func kubeVIPPod() string { - podBytes, err := yaml.Marshal(kubeVipPodSpec) - if err != nil { - panic(err) - } - return string(podBytes) -} diff --git a/packaging/flavorgen/flavors/kubevip/topology.go b/packaging/flavorgen/flavors/kubevip/topology.go index e0d81c46c6..f5df47e1fb 100644 --- a/packaging/flavorgen/flavors/kubevip/topology.go +++ b/packaging/flavorgen/flavors/kubevip/topology.go @@ -35,31 +35,19 @@ import ( // TopologyVariable returns the ClusterClass variable for kube-vip. func TopologyVariable() (*clusterv1.ClusterVariable, error) { - kubeVipPodYaml := kubeVIPPodYaml() - kubeVipPod, err := json.Marshal(kubeVipPodYaml) + out, err := json.Marshal(kubeVIPPodYAML()) if err != nil { - return nil, errors.Wrapf(err, "failed to json-encode variable kubeVipPod: %q", kubeVipPodYaml) + return nil, errors.Wrapf(err, "failed to json-encode variable kubeVipPod") } return &clusterv1.ClusterVariable{ Name: "kubeVipPodManifest", Value: apiextensionsv1.JSON{ - Raw: kubeVipPod, + Raw: out, }, }, nil } -// TopologyKubeVipPod returns the ClusterClass patch for kube-vip. -func TopologyKubeVipPod() ([]byte, error) { - kubeVipPodYaml := kubeVIPPodYaml() - kubeVipPod, err := json.Marshal(kubeVipPodYaml) - if err != nil { - return nil, errors.Wrapf(err, "failed to json-encode variable kubeVipPod: %q", kubeVipPodYaml) - } - - return kubeVipPod, nil -} - // TopologyPatch returns the ClusterClass patch for kube-vip. func TopologyPatch() clusterv1.ClusterClassPatch { patches := []clusterv1.JSONPatch{} diff --git a/templates/cluster-template-ignition.yaml b/templates/cluster-template-ignition.yaml index 415f02702a..03ed599ee3 100644 --- a/templates/cluster-template-ignition.yaml +++ b/templates/cluster-template-ignition.yaml @@ -89,29 +89,41 @@ spec: - args: - manager env: - - name: cp_enable + - name: vip_arp value: "true" - - name: vip_interface - value: ${VIP_NETWORK_INTERFACE:=""} - - name: address - value: ${CONTROL_PLANE_ENDPOINT_IP} - name: port value: "6443" - - name: vip_arp + - name: vip_interface + value: ${VIP_NETWORK_INTERFACE:=""} + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: svc_election value: "true" - name: vip_leaderelection value: "true" + - name: vip_leasename + value: plndr-cp-lock - name: vip_leaseduration value: "15" - name: vip_renewdeadline value: "10" - name: vip_retryperiod value: "2" - - name: svc_enable - value: "true" - - name: svc_election - value: "true" - image: ghcr.io/kube-vip/kube-vip:v0.6.3 + - name: address + value: ${CONTROL_PLANE_ENDPOINT_IP} + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.6.4 imagePullPolicy: IfNotPresent name: kube-vip resources: {} @@ -129,7 +141,6 @@ spec: volumes: - hostPath: path: /etc/kubernetes/admin.conf - type: File name: kubeconfig - hostPath: path: /etc/kube-vip.hosts diff --git a/templates/cluster-template-node-ipam.yaml b/templates/cluster-template-node-ipam.yaml index 92f74a2233..8423aabc1d 100644 --- a/templates/cluster-template-node-ipam.yaml +++ b/templates/cluster-template-node-ipam.yaml @@ -126,29 +126,41 @@ spec: - args: - manager env: - - name: cp_enable + - name: vip_arp value: "true" - - name: vip_interface - value: ${VIP_NETWORK_INTERFACE:=""} - - name: address - value: ${CONTROL_PLANE_ENDPOINT_IP} - name: port value: "6443" - - name: vip_arp + - name: vip_interface + value: ${VIP_NETWORK_INTERFACE:=""} + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: svc_election value: "true" - name: vip_leaderelection value: "true" + - name: vip_leasename + value: plndr-cp-lock - name: vip_leaseduration value: "15" - name: vip_renewdeadline value: "10" - name: vip_retryperiod value: "2" - - name: svc_enable - value: "true" - - name: svc_election - value: "true" - image: ghcr.io/kube-vip/kube-vip:v0.6.3 + - name: address + value: ${CONTROL_PLANE_ENDPOINT_IP} + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.6.4 imagePullPolicy: IfNotPresent name: kube-vip resources: {} @@ -166,7 +178,6 @@ spec: volumes: - hostPath: path: /etc/kubernetes/admin.conf - type: File name: kubeconfig - hostPath: path: /etc/kube-vip.hosts diff --git a/templates/cluster-template-topology.yaml b/templates/cluster-template-topology.yaml index a1e5d7e961..bc9d367a0c 100644 --- a/templates/cluster-template-topology.yaml +++ b/templates/cluster-template-topology.yaml @@ -23,6 +23,7 @@ spec: apiVersion: v1 kind: Pod metadata: + creationTimestamp: null name: kube-vip namespace: kube-system spec: @@ -30,29 +31,41 @@ spec: - args: - manager env: - - name: cp_enable + - name: vip_arp value: "true" - - name: vip_interface - value: ${VIP_NETWORK_INTERFACE:=""} - - name: address - value: ${CONTROL_PLANE_ENDPOINT_IP} - name: port value: "6443" - - name: vip_arp + - name: vip_interface + value: ${VIP_NETWORK_INTERFACE:=""} + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: svc_election value: "true" - name: vip_leaderelection value: "true" + - name: vip_leasename + value: plndr-cp-lock - name: vip_leaseduration value: "15" - name: vip_renewdeadline value: "10" - name: vip_retryperiod value: "2" - - name: svc_enable - value: "true" - - name: svc_election - value: "true" - image: ghcr.io/kube-vip/kube-vip:v0.6.3 + - name: address + value: ${CONTROL_PLANE_ENDPOINT_IP} + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.6.4 imagePullPolicy: IfNotPresent name: kube-vip resources: {} @@ -70,12 +83,12 @@ spec: volumes: - hostPath: path: /etc/kubernetes/admin.conf - type: File name: kubeconfig - hostPath: path: /etc/kube-vip.hosts type: File name: etchosts + status: {} - name: controlPlaneIpAddr value: ${CONTROL_PLANE_ENDPOINT_IP} - name: credsSecretName diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index 84aaf0c5f1..b0856905c2 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -116,29 +116,41 @@ spec: - args: - manager env: - - name: cp_enable + - name: vip_arp value: "true" - - name: vip_interface - value: ${VIP_NETWORK_INTERFACE:=""} - - name: address - value: ${CONTROL_PLANE_ENDPOINT_IP} - name: port value: "6443" - - name: vip_arp + - name: vip_interface + value: ${VIP_NETWORK_INTERFACE:=""} + - name: vip_cidr + value: "32" + - name: cp_enable + value: "true" + - name: cp_namespace + value: kube-system + - name: vip_ddns + value: "false" + - name: svc_enable + value: "true" + - name: svc_leasename + value: plndr-svcs-lock + - name: svc_election value: "true" - name: vip_leaderelection value: "true" + - name: vip_leasename + value: plndr-cp-lock - name: vip_leaseduration value: "15" - name: vip_renewdeadline value: "10" - name: vip_retryperiod value: "2" - - name: svc_enable - value: "true" - - name: svc_election - value: "true" - image: ghcr.io/kube-vip/kube-vip:v0.6.3 + - name: address + value: ${CONTROL_PLANE_ENDPOINT_IP} + - name: prometheus_server + value: :2112 + image: ghcr.io/kube-vip/kube-vip:v0.6.4 imagePullPolicy: IfNotPresent name: kube-vip resources: {} @@ -156,7 +168,6 @@ spec: volumes: - hostPath: path: /etc/kubernetes/admin.conf - type: File name: kubeconfig - hostPath: path: /etc/kube-vip.hosts