diff --git a/api/v1alpha1/ionoscloudmachine_types.go b/api/v1alpha1/ionoscloudmachine_types.go index 0a4abae3..bdc6b52e 100644 --- a/api/v1alpha1/ionoscloudmachine_types.go +++ b/api/v1alpha1/ionoscloudmachine_types.go @@ -240,6 +240,17 @@ type ImageSpec struct { Selector *ImageSelector `json:"selector,omitempty"` } +// ImageSelectorResolutionPolicy defines what action to take when the image selector has ambiguous results. +// +kubebuilder:validation:Enum=Exact;Newest +type ImageSelectorResolutionPolicy string + +const ( + // ResolutionPolicyExact only succeeds if the image selector resolves to exactly 1 image. + ResolutionPolicyExact ImageSelectorResolutionPolicy = "Exact" + // ResolutionPolicyNewest uses the newest entry if the image selector resolves to more than 1 image. + ResolutionPolicyNewest ImageSelectorResolutionPolicy = "Newest" +) + // ImageSelector defines label selectors for looking up images. type ImageSelector struct { // MatchLabels is a map of key/value pairs. @@ -247,6 +258,14 @@ type ImageSelector struct { //+kubebuilder:validation:MinProperties=1 MatchLabels map[string]string `json:"matchLabels"` + // ResolutionPolicy controls the lookup behavior. + // The default policy 'Exact' will raise an error if the selector resolves to more than 1 image. + // Use policy 'Newest' to select the newest image instead. + // + //+kubebuilder:default=`Exact` + //+optional + ResolutionPolicy ImageSelectorResolutionPolicy `json:"resolutionPolicy,omitempty"` + // UseMachineVersion indicates whether to use the parent Machine's version field to look up image names. // Enabled by default. // diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudmachines.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudmachines.yaml index 2072c4d2..f5d511cf 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudmachines.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudmachines.yaml @@ -213,6 +213,16 @@ spec: description: MatchLabels is a map of key/value pairs. minProperties: 1 type: object + resolutionPolicy: + default: Exact + description: |- + ResolutionPolicy controls the lookup behavior. + The default policy 'Exact' will raise an error if the selector resolves to more than 1 image. + Use policy 'Newest' to select the newest image instead. + enum: + - Exact + - Newest + type: string useMachineVersion: default: true description: |- diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudmachinetemplates.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudmachinetemplates.yaml index 2565b2a7..12a837ac 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudmachinetemplates.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_ionoscloudmachinetemplates.yaml @@ -234,6 +234,16 @@ spec: pairs. minProperties: 1 type: object + resolutionPolicy: + default: Exact + description: |- + ResolutionPolicy controls the lookup behavior. + The default policy 'Exact' will raise an error if the selector resolves to more than 1 image. + Use policy 'Newest' to select the newest image instead. + enum: + - Exact + - Newest + type: string useMachineVersion: default: true description: |- diff --git a/internal/service/cloud/image.go b/internal/service/cloud/image.go index a67df5c7..04ba59b8 100644 --- a/internal/service/cloud/image.go +++ b/internal/service/cloud/image.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "slices" "strings" sdk "github.com/ionos-cloud/sdk-go/v6" @@ -70,8 +71,20 @@ func (s *Service) lookupImageID(ctx context.Context, ms *scope.Machine) (string, images = filterImagesByName(images, version) } - if n := len(images); n != 1 { - return "", imageMatchError{imageIDs: getImageIDs(images), selector: imageSpec.Selector} + if len(images) == 0 { + return "", imageMatchError{selector: imageSpec.Selector} + } + + switch imageSpec.Selector.ResolutionPolicy { + case infrav1.ResolutionPolicyExact: + if len(images) > 1 { + return "", imageMatchError{imageIDs: getImageIDs(images), selector: imageSpec.Selector} + } + case infrav1.ResolutionPolicyNewest: + slices.SortFunc(images, func(lhs, rhs *sdk.Image) int { + // swap lhs and rhs to produce reverse order + return rhs.Metadata.CreatedDate.Compare(lhs.Metadata.CreatedDate.Time) + }) } return ptr.Deref(images[0].GetId(), ""), nil diff --git a/internal/service/cloud/image_test.go b/internal/service/cloud/image_test.go index 4db74623..4a353c05 100644 --- a/internal/service/cloud/image_test.go +++ b/internal/service/cloud/image_test.go @@ -21,6 +21,7 @@ import ( "fmt" "slices" "testing" + "time" "github.com/go-logr/logr" sdk "github.com/ionos-cloud/sdk-go/v6" @@ -47,6 +48,7 @@ func (s *imageTestSuite) SetupTest() { MatchLabels: map[string]string{ "test": "image", }, + ResolutionPolicy: infrav1.ResolutionPolicyExact, } } @@ -121,7 +123,7 @@ func (s *imageTestSuite) TestLookupImageIgnoreMissingMachineVersion() { s.Equal("image-1", imageID) } -func (s *imageTestSuite) TestLookupImageOK() { +func (s *imageTestSuite) TestLookupImageExactOK() { s.ionosClient.EXPECT().ListLabels(s.ctx).Return( []sdk.Label{ makeTestLabel("image", "image-1", "test", "image"), @@ -135,10 +137,37 @@ func (s *imageTestSuite) TestLookupImageOK() { s.Equal("image-1", imageID) } +func (s *imageTestSuite) TestLookupImageNewestOK() { + s.ionosClient.EXPECT().ListLabels(s.ctx).Return( + []sdk.Label{ + makeTestLabel("image", "image-1", "test", "image"), + makeTestLabel("image", "image-2", "test", "image"), + makeTestLabel("image", "image-3", "test", "image"), + }, nil, + ).Once() + s.ionosClient.EXPECT().GetDatacenterLocationByID(s.ctx, s.infraMachine.Spec.DatacenterID).Return("loc", nil).Once() + baseTime := time.Now().Round(time.Second) + s.ionosClient.EXPECT().GetImage(s.ctx, "image-1").Return(s.makeTestImageWithDate("image-1", "img-ver1-", "loc", baseTime), nil).Once() + s.ionosClient.EXPECT().GetImage(s.ctx, "image-2").Return(s.makeTestImageWithDate("image-2", "img-ver2-", "loc", baseTime.Add(2*time.Minute)), nil).Once() + s.ionosClient.EXPECT().GetImage(s.ctx, "image-3").Return(s.makeTestImageWithDate("image-3", "img-ver3-", "loc", baseTime.Add(1*time.Minute)), nil).Once() + + s.infraMachine.Spec.Disk.Image.Selector.ResolutionPolicy = infrav1.ResolutionPolicyNewest + + imageID, err := s.service.lookupImageID(s.ctx, s.machineScope) + s.NoError(err) + s.Equal("image-2", imageID) +} + func (s *imageTestSuite) makeTestImage(id, namePrefix, location string) *sdk.Image { return makeTestImage(id, namePrefix+*s.capiMachine.Spec.Version, location) } +func (s *imageTestSuite) makeTestImageWithDate(id, namePrefix, location string, createdDate time.Time) *sdk.Image { + img := s.makeTestImage(id, namePrefix, location) + img.Metadata.CreatedDate = &sdk.IonosTime{Time: createdDate} + return img +} + func TestFilterImagesByName(t *testing.T) { images := []*sdk.Image{ makeTestImage("image-1", "img-foo-v1.1.qcow2", "test"), @@ -194,7 +223,8 @@ func TestLookupImagesBySelector(t *testing.T) { func makeTestImage(id, name, location string) *sdk.Image { return &sdk.Image{ - Id: &id, + Id: &id, + Metadata: &sdk.DatacenterElementMetadata{}, Properties: &sdk.ImageProperties{ Name: &name, Location: &location, diff --git a/templates/cluster-template-auto-image.yaml b/templates/cluster-template-auto-image.yaml index 48f1deb7..bc52dacd 100644 --- a/templates/cluster-template-auto-image.yaml +++ b/templates/cluster-template-auto-image.yaml @@ -291,6 +291,7 @@ spec: disk: image: selector: + resolutionPolicy: ${IONOSCLOUD_IMAGE_RESOLUTION_POLICY:-Newest} matchLabels: ${IONOSCLOUD_IMAGE_LABEL_KEY}: ${IONOSCLOUD_IMAGE_LABEL_VALUE} --- @@ -336,6 +337,7 @@ spec: disk: image: selector: + resolutionPolicy: ${IONOSCLOUD_IMAGE_RESOLUTION_POLICY:-Newest} matchLabels: ${IONOSCLOUD_IMAGE_LABEL_KEY}: ${IONOSCLOUD_IMAGE_LABEL_VALUE} --- diff --git a/test/e2e/capic_test.go b/test/e2e/capic_test.go index dea8ccac..2a9e0537 100644 --- a/test/e2e/capic_test.go +++ b/test/e2e/capic_test.go @@ -92,6 +92,7 @@ var _ = Describe("Should be able to create a cluster with 1 control-plane and 1 E2EConfig: e2eConfig, ClusterctlConfigPath: clusterctlConfigPath, BootstrapClusterProxy: bootstrapClusterProxy, + Flavor: "image-selector", ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, PostNamespaceCreated: cloudEnv.createCredentialsSecretPNC, diff --git a/test/e2e/config/ionoscloud.yaml b/test/e2e/config/ionoscloud.yaml index 0a356075..9a7fa42c 100644 --- a/test/e2e/config/ionoscloud.yaml +++ b/test/e2e/config/ionoscloud.yaml @@ -65,12 +65,13 @@ providers: - sourcePath: "../../../metadata.yaml" - sourcePath: "../data/infrastructure-ionoscloud/cluster-template.yaml" - sourcePath: "../data/infrastructure-ionoscloud/cluster-template-ipam.yaml" + - sourcePath: "../data/infrastructure-ionoscloud/cluster-template-image-selector.yaml" variables: # Default variables for the e2e test; those values could be overridden via env variables, thus # allowing the same e2e config file to be re-used in different Prow jobs e.g. each one with a K8s version permutation. # The following Kubernetes versions should be the latest versions with already published kindest/node images. # This avoids building node images in the default case which improves the test duration significantly. - KUBERNETES_VERSION: "v1.29.2" + KUBERNETES_VERSION: "v1.30.6" CNI: "./data/cni/calico.yaml" KUBETEST_CONFIGURATION: "./data/kubetest/conformance.yaml" CLUSTER_NAME: "e2e-cluster-${RANDOM}" diff --git a/test/e2e/data/infrastructure-ionoscloud/cluster-template-image-selector.yaml b/test/e2e/data/infrastructure-ionoscloud/cluster-template-image-selector.yaml new file mode 100644 index 00000000..3a757fdc --- /dev/null +++ b/test/e2e/data/infrastructure-ionoscloud/cluster-template-image-selector.yaml @@ -0,0 +1,454 @@ +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + cni: "${CLUSTER_NAME}-crs-0" +spec: + clusterNetwork: + pods: + cidrBlocks: ["192.168.0.0/16"] + infrastructureRef: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: IonosCloudCluster + name: "${CLUSTER_NAME}" + controlPlaneRef: + kind: KubeadmControlPlane + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + name: "${CLUSTER_NAME}-control-plane" +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: IonosCloudCluster +metadata: + name: "${CLUSTER_NAME}" +spec: + controlPlaneEndpoint: + host: ${CONTROL_PLANE_ENDPOINT_HOST:-${CONTROL_PLANE_ENDPOINT_IP}} + port: ${CONTROL_PLANE_ENDPOINT_PORT:-6443} + location: ${CONTROL_PLANE_ENDPOINT_LOCATION} + credentialsRef: + name: "ionoscloud-credentials" +--- +kind: KubeadmControlPlane +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + replicas: ${CONTROL_PLANE_MACHINE_COUNT} + machineTemplate: + infrastructureRef: + kind: IonosCloudMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + name: "${CLUSTER_NAME}-control-plane" + kubeadmConfigSpec: + users: + - name: root + sshAuthorizedKeys: [${IONOSCLOUD_MACHINE_SSH_KEYS}] + ntp: + enabled: true + servers: + - 0.de.pool.ntp.org + - 1.de.pool.ntp.org + - 2.de.pool.ntp.org + - 3.de.pool.ntp.org + files: + - path: /etc/ssh/sshd_config.d/ssh-audit_hardening.conf + owner: root:root + permissions: '0644' + content: | + # Restrict key exchange, cipher, and MAC algorithms, as per sshaudit.com + # hardening guide. + KexAlgorithms sntrup761x25519-sha512@openssh.com,curve25519-sha256,curve25519-sha256@libssh.org,gss-curve25519-sha256-,diffie-hellman-group16-sha512,gss-group16-sha512-,diffie-hellman-group18-sha512,diffie-hellman-group-exchange-sha256 + Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr + MACs hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,umac-128-etm@openssh.com + HostKeyAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512,rsa-sha2-256 + CASignatureAlgorithms sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512,rsa-sha2-256 + GSSAPIKexAlgorithms gss-curve25519-sha256-,gss-group16-sha512- + HostbasedAcceptedAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-256 + PubkeyAcceptedAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-256 + - path: /etc/sysctl.d/k8s.conf + content: | + fs.inotify.max_user_watches = 65536 + net.netfilter.nf_conntrack_max = 1000000 + - path: /etc/modules-load.d/k8s.conf + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + ip_vs_sed + # Crictl config + - path: /etc/crictl.yaml + content: | + runtime-endpoint: unix:///run/containerd/containerd.sock + timeout: 10 + - path: /etc/kubernetes/manifests/kube-vip.yaml + owner: root:root + content: | + apiVersion: v1 + kind: Pod + metadata: + name: kube-vip + namespace: kube-system + spec: + containers: + - args: + - manager + env: + - name: cp_enable + value: "true" + - name: vip_interface + value: ${VIP_NETWORK_INTERFACE=""} + - name: address + value: ${CONTROL_PLANE_ENDPOINT_IP} + - name: port + value: "${CONTROL_PLANE_ENDPOINT_PORT:-6443}" + - name: vip_arp + value: "true" + - name: vip_leaderelection + value: "true" + - name: vip_leaseduration + value: "15" + - name: vip_renewdeadline + value: "10" + - name: vip_retryperiod + value: "2" + image: ghcr.io/kube-vip/kube-vip:v0.7.1 + imagePullPolicy: IfNotPresent + name: kube-vip + resources: {} + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + volumeMounts: + - mountPath: /etc/kubernetes/admin.conf + name: kubeconfig + hostAliases: + - hostnames: + - kubernetes + - localhost + ip: 127.0.0.1 + hostNetwork: true + volumes: + - hostPath: + path: /etc/kubernetes/admin.conf + type: FileOrCreate + name: kubeconfig + status: {} + - path: /etc/kube-vip-prepare.sh + content: | + #!/bin/bash + + # Copyright 2020 The Kubernetes Authors. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + set -e + + # Configure the workaround required for kubeadm init with kube-vip: + # xref: https://github.com/kube-vip/kube-vip/issues/684 + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "$KUBEADM_MINOR" -lt "29" ]]; then + exit 0 + fi + + IS_KUBEADM_INIT="false" + + # cloud-init kubeadm init + if [[ -f /run/kubeadm/kubeadm.yaml ]]; then + IS_KUBEADM_INIT="true" + fi + + # ignition kubeadm init + if [[ -f /etc/kubeadm.sh ]] && grep -q -e "kubeadm init" /etc/kubeadm.sh; then + IS_KUBEADM_INIT="true" + fi + + if [[ "$IS_KUBEADM_INIT" == "true" ]]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + fi + owner: root:root + permissions: "0700" + + # CSI Metadata config + - content: | + { + "datacenter-id": "${IONOSCLOUD_DATACENTER_ID}" + } + owner: root:root + path: /etc/ie-csi/cfg.json + permissions: '0644' + + - content: | + #!/bin/bash + set -e + + # Nothing to do for kubernetes < v1.29 + KUBEADM_MINOR="$(kubeadm version -o short | cut -d '.' -f 2)" + if [[ "$KUBEADM_MINOR" -lt "29" ]]; then + exit 0 + fi + + NODE_IPv4_ADDRESS=$(ip -j addr show dev ens6 | jq -r '.[].addr_info[] | select(.family == "inet") | select(.scope=="global") | select(.dynamic) | .local') + if [[ $NODE_IPv4_ADDRESS ]]; then + sed -i '$ s/$/ --node-ip '"$NODE_IPv4_ADDRESS"'/' /etc/default/kubelet + fi + # IPv6 currently not set, the ip is not set then this runs. Needs to be waited for. + NODE_IPv6_ADDRESS=$(ip -j addr show dev ens6 | jq -r '.[].addr_info[] | select(.family == "inet6") | select(.scope=="global") | .local') + if [[ $NODE_IPv6_ADDRESS ]]; then + sed -i '$ s/$/ --node-ip '"$NODE_IPv6_ADDRESS"'/' /etc/default/kubelet + fi + owner: root:root + path: /etc/set-node-ip.sh + permissions: '0700' + + preKubeadmCommands: + - systemctl restart systemd-networkd.service systemd-modules-load.service systemd-journald containerd + # disable swap + - swapoff -a + - sed -i '/ swap / s/^/#/' /etc/fstab + - sysctl --system + - /etc/kube-vip-prepare.sh + # workaround 1.29 IP issue + - /etc/set-node-ip.sh + postKubeadmCommands: + - > + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' \ + /etc/kubernetes/manifests/kube-vip.yaml + - > + systemctl disable --now udisks2 multipathd motd-news.timer fwupd-refresh.timer + packagekit ModemManager snapd snapd.socket snapd.apparmor snapd.seeded + # INFO(schegi-ionos): We decided to not remove this for now, since removing this would require the ccm to be installed for cluster-api + # to continue after the first node. + - export system_uuid=$(kubectl --kubeconfig /etc/kubernetes/kubelet.conf get node $(hostname) -ojsonpath='{..systemUUID }') + - > + kubectl --kubeconfig /etc/kubernetes/kubelet.conf + patch node $(hostname) + --type strategic -p '{"spec": {"providerID": "ionos://'$${system_uuid}'"}}' + - rm /etc/ssh/ssh_host_* + - ssh-keygen -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key -N "" + - ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -N "" + - sed -i 's/^\#HostKey \/etc\/ssh\/ssh_host_\(rsa\|ed25519\)_key$/HostKey \/etc\/ssh\/ssh_host_\1_key/g' /etc/ssh/sshd_config + - awk '$5 >= 3071' /etc/ssh/moduli > /etc/ssh/moduli.safe + - mv /etc/ssh/moduli.safe /etc/ssh/moduli + - iptables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --set + - iptables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --update --seconds 10 --hitcount 10 -j DROP + - ip6tables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --set + - ip6tables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --update --seconds 10 --hitcount 10 -j DROP + - apt-get update + - DEBIAN_FRONTEND=noninteractive apt-get install -q -y netfilter-persistent iptables-persistent + - service netfilter-persistent save + - systemctl restart sshd + initConfiguration: + localAPIEndpoint: + bindPort: ${CONTROL_PLANE_ENDPOINT_PORT:-6443} + nodeRegistration: + kubeletExtraArgs: + # use cloud-provider: external when using a CCM + cloud-provider: "" + joinConfiguration: + nodeRegistration: + criSocket: unix:///run/containerd/containerd.sock + kubeletExtraArgs: + # use cloud-provider: external when using a CCM + cloud-provider: "" + version: "${KUBERNETES_VERSION}" +--- +kind: IonosCloudMachineTemplate +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +metadata: + name: "${CLUSTER_NAME}-control-plane" +spec: + template: + spec: + datacenterID: ${IONOSCLOUD_DATACENTER_ID} + numCores: ${IONOSCLOUD_MACHINE_NUM_CORES:-4} + memoryMB: ${IONOSCLOUD_MACHINE_MEMORY_MB:-8192} + disk: + image: + selector: + resolutionPolicy: Newest + matchLabels: + ${IONOSCLOUD_IMAGE_LABEL_KEY:-e2etest}: ${IONOSCLOUD_IMAGE_LABEL_VALUE:-capic} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + name: "${CLUSTER_NAME}-workers" + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" +spec: + clusterName: "${CLUSTER_NAME}" + replicas: ${WORKER_MACHINE_COUNT} + selector: + matchLabels: + template: + metadata: + labels: + cluster.x-k8s.io/cluster-name: "${CLUSTER_NAME}" + node-role.kubernetes.io/node: "" + spec: + clusterName: "${CLUSTER_NAME}" + version: "${KUBERNETES_VERSION}" + bootstrap: + configRef: + name: "${CLUSTER_NAME}-worker" + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + infrastructureRef: + name: "${CLUSTER_NAME}-worker" + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: IonosCloudMachineTemplate +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: IonosCloudMachineTemplate +metadata: + name: "${CLUSTER_NAME}-worker" +spec: + template: + spec: + datacenterID: ${IONOSCLOUD_DATACENTER_ID} + numCores: ${IONOSCLOUD_MACHINE_NUM_CORES:-2} + memoryMB: ${IONOSCLOUD_MACHINE_MEMORY_MB:-4096} + disk: + image: + selector: + resolutionPolicy: Newest + matchLabels: + ${IONOSCLOUD_IMAGE_LABEL_KEY:-e2etest}: ${IONOSCLOUD_IMAGE_LABEL_VALUE:-capic} +--- +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: "${CLUSTER_NAME}-worker" +spec: + template: + spec: + users: + - name: root + sshAuthorizedKeys: [${IONOSCLOUD_MACHINE_SSH_KEYS}] + ntp: + enabled: true + servers: + - 0.de.pool.ntp.org + - 1.de.pool.ntp.org + - 2.de.pool.ntp.org + - 3.de.pool.ntp.org + files: + - path: /etc/ssh/sshd_config.d/ssh-audit_hardening.conf + owner: root:root + permissions: '0644' + content: | + # Restrict key exchange, cipher, and MAC algorithms, as per sshaudit.com + # hardening guide. + KexAlgorithms sntrup761x25519-sha512@openssh.com,curve25519-sha256,curve25519-sha256@libssh.org,gss-curve25519-sha256-,diffie-hellman-group16-sha512,gss-group16-sha512-,diffie-hellman-group18-sha512,diffie-hellman-group-exchange-sha256 + Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com,aes256-ctr,aes192-ctr,aes128-ctr + MACs hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,umac-128-etm@openssh.com + HostKeyAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512,rsa-sha2-256 + CASignatureAlgorithms sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512,rsa-sha2-256 + GSSAPIKexAlgorithms gss-curve25519-sha256-,gss-group16-sha512- + HostbasedAcceptedAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-256 + PubkeyAcceptedAlgorithms sk-ssh-ed25519-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,sk-ssh-ed25519@openssh.com,ssh-ed25519,rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-256-cert-v01@openssh.com,rsa-sha2-256 + - path: /etc/sysctl.d/k8s.conf + content: | + fs.inotify.max_user_watches = 65536 + net.netfilter.nf_conntrack_max = 1000000 + - path: /etc/modules-load.d/k8s.conf + content: | + ip_vs + ip_vs_rr + ip_vs_wrr + ip_vs_sh + ip_vs_sed + # Crictl config + - path: /etc/crictl.yaml + content: | + runtime-endpoint: unix:///run/containerd/containerd.sock + timeout: 10 + # CSI Metadata config + - content: | + { + "datacenter-id": "${IONOSCLOUD_DATACENTER_ID}" + } + owner: root:root + path: /etc/ie-csi/cfg.json + permissions: '0644' + preKubeadmCommands: + - systemctl restart systemd-networkd.service systemd-modules-load.service systemd-journald containerd + # disable swap + - swapoff -a + - sed -i '/ swap / s/^/#/' /etc/fstab + - sysctl --system + postKubeadmCommands: + - > + systemctl disable --now udisks2 multipathd motd-news.timer fwupd-refresh.timer + packagekit ModemManager snapd snapd.socket snapd.apparmor snapd.seeded + # INFO(schegi-ionos): We decided to not remove this for now, since removing this would require the ccm to be + # installed for cluster-api to continue after the first node. + - export system_uuid=$(kubectl --kubeconfig /etc/kubernetes/kubelet.conf get node $(hostname) -ojsonpath='{..systemUUID }') + - > + kubectl --kubeconfig /etc/kubernetes/kubelet.conf + patch node $(hostname) + --type strategic -p '{"spec": {"providerID": "ionos://'$${system_uuid}'"}}' + - rm /etc/ssh/ssh_host_* + - ssh-keygen -t rsa -b 4096 -f /etc/ssh/ssh_host_rsa_key -N "" + - ssh-keygen -t ed25519 -f /etc/ssh/ssh_host_ed25519_key -N "" + - sed -i 's/^\#HostKey \/etc\/ssh\/ssh_host_\(rsa\|ed25519\)_key$/HostKey \/etc\/ssh\/ssh_host_\1_key/g' /etc/ssh/sshd_config + - awk '$5 >= 3071' /etc/ssh/moduli > /etc/ssh/moduli.safe + - mv /etc/ssh/moduli.safe /etc/ssh/moduli + - iptables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --set + - iptables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --update --seconds 10 --hitcount 10 -j DROP + - ip6tables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --set + - ip6tables -I INPUT -p tcp --dport 22 -m state --state NEW -m recent --update --seconds 10 --hitcount 10 -j DROP + - apt-get update + - DEBIAN_FRONTEND=noninteractive apt-get install -q -y netfilter-persistent iptables-persistent + - service netfilter-persistent save + - systemctl restart sshd + joinConfiguration: + nodeRegistration: + kubeletExtraArgs: + # use cloud-provider: external when using a CCM + cloud-provider: "" + criSocket: unix:///run/containerd/containerd.sock +--- +# ConfigMap object referenced by the ClusterResourceSet object and with +# the CNI resource defined in the test config file +apiVersion: v1 +kind: ConfigMap +metadata: + name: "cni-${CLUSTER_NAME}-crs-0" +data: ${CNI_RESOURCES} +--- +# ClusterResourceSet object with +# a selector that targets all the Cluster with label cni=${CLUSTER_NAME}-crs-0 +apiVersion: addons.cluster.x-k8s.io/v1beta1 +kind: ClusterResourceSet +metadata: + labels: + cluster.x-k8s.io/cluster-name: '${CLUSTER_NAME}' + name: "${CLUSTER_NAME}-crs-0" +spec: + strategy: ApplyOnce + clusterSelector: + matchLabels: + cni: "${CLUSTER_NAME}-crs-0" + resources: + - name: "cni-${CLUSTER_NAME}-crs-0" + kind: ConfigMap diff --git a/test/e2e/env_test.go b/test/e2e/env_test.go index b80f37dc..310fc2ac 100644 --- a/test/e2e/env_test.go +++ b/test/e2e/env_test.go @@ -79,7 +79,7 @@ func (e *ionosCloudEnv) setup() { } func (e *ionosCloudEnv) teardown() { - if !skipCleanup { + if !skipCleanup && e.api != nil { By("Deleting environment resources") By("Requesting the deletion of the data center") diff --git a/test/e2e/k8s_conformance_test.go b/test/e2e/k8s_conformance_test.go index 1a70f7d0..99e5d9e3 100644 --- a/test/e2e/k8s_conformance_test.go +++ b/test/e2e/k8s_conformance_test.go @@ -34,6 +34,7 @@ var _ = Describe("When testing K8S conformance", Label("Conformance"), func() { BootstrapClusterProxy: bootstrapClusterProxy, ArtifactFolder: artifactFolder, SkipCleanup: skipCleanup, + Flavor: "image-selector", PostNamespaceCreated: cloudEnv.createCredentialsSecretPNC, } })