From e87ff14a76ccb8015b06b195d67d4bec81b9bc2b Mon Sep 17 00:00:00 2001 From: "Lubomir I. Ivanov" Date: Wed, 28 Feb 2024 19:45:09 +0200 Subject: [PATCH] kinder: add e2e test for WaitForAllControlPlaneComponents --- kinder/ci/tools/update-workflows/config.yaml | 12 ++ ...wait-for-all-control-plane-components.yaml | 42 ++++++ ...or-all-control-plane-components-tasks.yaml | 121 +++++++++++++++++ ...wait-for-all-control-plane-components.yaml | 10 ++ ...r-all-control-plane-components-latest.yaml | 11 ++ ...or-all-control-plane-components-tasks.yaml | 122 ++++++++++++++++++ 6 files changed, 318 insertions(+) create mode 100644 kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-wait-for-all-control-plane-components.yaml create mode 100644 kinder/ci/tools/update-workflows/templates/workflows/wait-for-all-control-plane-components-tasks.yaml create mode 100644 kinder/ci/tools/update-workflows/templates/workflows/wait-for-all-control-plane-components.yaml create mode 100644 kinder/ci/workflows/wait-for-all-control-plane-components-latest.yaml create mode 100644 kinder/ci/workflows/wait-for-all-control-plane-components-tasks.yaml diff --git a/kinder/ci/tools/update-workflows/config.yaml b/kinder/ci/tools/update-workflows/config.yaml index 9e3893c6..735a9216 100644 --- a/kinder/ci/tools/update-workflows/config.yaml +++ b/kinder/ci/tools/update-workflows/config.yaml @@ -237,3 +237,15 @@ jobGroups: - ./templates/workflows/super-admin-tasks.yaml jobs: - kubernetesVersion: latest + +- name: wait-for-all-control-plane-components + testInfraJobSpec: + targetFile: kubeadm-kinder-wait-for-all-control-plane-components.yaml + template: ./templates/testinfra/kubeadm-kinder-wait-for-all-control-plane-components.yaml + kinderWorkflowSpec: + targetFile: wait-for-all-control-plane-components-{{ .KubernetesVersion }}.yaml + template: ./templates/workflows/wait-for-all-control-plane-components.yaml + additionalFiles: + - ./templates/workflows/wait-for-all-control-plane-components-tasks.yaml + jobs: + - kubernetesVersion: latest diff --git a/kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-wait-for-all-control-plane-components.yaml b/kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-wait-for-all-control-plane-components.yaml new file mode 100644 index 00000000..0a271f76 --- /dev/null +++ b/kinder/ci/tools/update-workflows/templates/testinfra/kubeadm-kinder-wait-for-all-control-plane-components.yaml @@ -0,0 +1,42 @@ +- name: ci-kubernetes-e2e-kubeadm-kinder-wait-for-all-control-plane-components-{{ dashVer .KubernetesVersion }} + cluster: k8s-infra-prow-build + interval: {{ .JobInterval }} + decorate: true + labels: + preset-dind-enabled: "true" + preset-kind-volume-mounts: "true" + annotations: + testgrid-dashboards: sig-cluster-lifecycle-kubeadm + testgrid-tab-name: kubeadm-kinder-wait-for-all-control-plane-components-{{ dashVer .KubernetesVersion }} + testgrid-alert-email: sig-cluster-lifecycle-kubeadm-alerts@kubernetes.io + description: "OWNER: sig-cluster-lifecycle (kinder); Uses kubeadm/kinder to create a cluster with the WaitForAllControlPlaneComponents feature gate" + testgrid-num-columns-recent: "20" +{{ .AlertAnnotations }} + decoration_config: + timeout: 60m + extra_refs: + - org: kubernetes + repo: kubernetes + base_ref: {{ branchFor .KubernetesVersion }} + path_alias: k8s.io/kubernetes + - org: kubernetes + repo: kubeadm + base_ref: main + path_alias: k8s.io/kubeadm + spec: + containers: + - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:{{ .TestInfraImage }}-{{ imageVer .KubernetesVersion }} + command: + - runner.sh + - "../kubeadm/kinder/ci/kinder-run.sh" + args: + - {{ .WorkflowFile }} + securityContext: + privileged: true + resources: + limits: + memory: "9000Mi" + cpu: 2000m + requests: + memory: "9000Mi" + cpu: 2000m diff --git a/kinder/ci/tools/update-workflows/templates/workflows/wait-for-all-control-plane-components-tasks.yaml b/kinder/ci/tools/update-workflows/templates/workflows/wait-for-all-control-plane-components-tasks.yaml new file mode 100644 index 00000000..517fab33 --- /dev/null +++ b/kinder/ci/tools/update-workflows/templates/workflows/wait-for-all-control-plane-components-tasks.yaml @@ -0,0 +1,121 @@ +# IMPORTANT! this workflow is imported by wait-for-all-control-plane-components* workflows. +version: 1 +summary: | + This workflow implements a sequence of tasks used to test kubeadm init and join with the WaitForAllControlPlaneComponents feature gate enabled. +vars: + # vars defines default values for variable used by tasks in this workflow; + # those values might be overridden when importing this files. + initVersion: v1.12.8 + upgradeVersion: v1.13.5 + controlPlaneNodes: 3 + workerNodes: 2 + baseImage: kindest/base:v20221102-76f15095 # has containerd + image: kindest/node:test + clusterName: kinder-wait-for-all-control-plane-components + kubeadmVerbosity: 6 +tasks: +- name: pull-base-image + description: | + pulls kindest/base image with docker in docker and all the prerequisites necessary for running kind(er) + cmd: docker + args: + - pull + - "{{ .vars.baseImage }}" +- name: add-kubernetes-versions + description: | + creates a node-image-variant by adding Kubernetes version "initVersion" + to be used when executing "kinder do kubeadm-init". + cmd: kinder + args: + - build + - node-image-variant + - --base-image={{ .vars.baseImage }} + - --image={{ .vars.image }} + - --with-init-artifacts={{ .vars.initVersion }} + - --loglevel=debug + timeout: 15m +- name: create-cluster + description: | + create a set of nodes ready for hosting the Kubernetes cluster + cmd: kinder + args: + - create + - cluster + - --name={{ .vars.clusterName }} + - --image={{ .vars.image }} + - --control-plane-nodes={{ .vars.controlPlaneNodes }} + - --worker-nodes={{ .vars.workerNodes }} + - --loglevel=debug + timeout: 5m +- name: init + description: | + Initializes the Kubernetes cluster with version "initVersion" + by starting the boostrap control-plane nodes + cmd: kinder + args: + - do + - kubeadm-init + - --name={{ .vars.clusterName }} + - --copy-certs=auto + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + - --kubeadm-feature-gate="WaitForAllControlPlaneComponents=true" + timeout: 5m +- name: join + description: | + Join the other nodes to the Kubernetes cluster + cmd: kinder + args: + - do + - kubeadm-join + - --name={{ .vars.clusterName }} + - --copy-certs=auto + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + timeout: 10m +- name: cluster-info + description: | + Runs cluster-info on the cluster before upgrade + cmd: kinder + args: + - do + - cluster-info + - --name={{ .vars.clusterName }} + - --loglevel=debug +- name: get-logs + description: | + Collects all the test logs + cmd: kinder + args: + - export + - logs + - --loglevel=debug + - --name={{ .vars.clusterName }} + - "{{ .env.ARTIFACTS }}" + force: true + timeout: 5m + # kind export log is know to be flaky, so we are temporary ignoring errors in order + # to make the test pass in case everything else passed + # see https://github.com/kubernetes-sigs/kind/issues/456 + ignoreError: true +- name: reset + description: | + Exec kubeadm reset + cmd: kinder + args: + - do + - kubeadm-reset + - --name={{ .vars.clusterName }} + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + force: true +- name: delete + description: | + Deletes the cluster + cmd: kinder + args: + - delete + - cluster + - --name={{ .vars.clusterName }} + - --loglevel=debug + force: true diff --git a/kinder/ci/tools/update-workflows/templates/workflows/wait-for-all-control-plane-components.yaml b/kinder/ci/tools/update-workflows/templates/workflows/wait-for-all-control-plane-components.yaml new file mode 100644 index 00000000..694b5601 --- /dev/null +++ b/kinder/ci/tools/update-workflows/templates/workflows/wait-for-all-control-plane-components.yaml @@ -0,0 +1,10 @@ +version: 1 +summary: | + This workflow tests if kubeadm {{ .InitVersion }} can "init" and "join" nodes with the feature gate + WaitForAllControlPlaneComponents feature enabled. The Kubernetes version is at {{ .KubernetesVersion }}. + test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-wait-for-all-control-plane-components-{{ dashVer .KubernetesVersion }} + config > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/{{ .TargetFile }} +vars: + initVersion: "\{\{ resolve `ci/{{ ciLabelFor .InitVersion }}` \}\}" +tasks: +- import: wait-for-all-control-plane-components-tasks.yaml diff --git a/kinder/ci/workflows/wait-for-all-control-plane-components-latest.yaml b/kinder/ci/workflows/wait-for-all-control-plane-components-latest.yaml new file mode 100644 index 00000000..2ccde86f --- /dev/null +++ b/kinder/ci/workflows/wait-for-all-control-plane-components-latest.yaml @@ -0,0 +1,11 @@ +# AUTOGENERATED by https://git.k8s.io/kubeadm/kinder/ci/tools/update-workflows +version: 1 +summary: | + This workflow tests if kubeadm latest can "init" and "join" nodes with the feature gate + WaitForAllControlPlaneComponents feature enabled. The Kubernetes version is at latest. + test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-wait-for-all-control-plane-components-latest + config > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/kubeadm-kinder-wait-for-all-control-plane-components.yaml +vars: + initVersion: "{{ resolve `ci/latest` }}" +tasks: +- import: wait-for-all-control-plane-components-tasks.yaml diff --git a/kinder/ci/workflows/wait-for-all-control-plane-components-tasks.yaml b/kinder/ci/workflows/wait-for-all-control-plane-components-tasks.yaml new file mode 100644 index 00000000..1e77805a --- /dev/null +++ b/kinder/ci/workflows/wait-for-all-control-plane-components-tasks.yaml @@ -0,0 +1,122 @@ +# AUTOGENERATED by https://git.k8s.io/kubeadm/kinder/ci/tools/update-workflows +# IMPORTANT! this workflow is imported by wait-for-all-control-plane-components* workflows. +version: 1 +summary: | + This workflow implements a sequence of tasks used to test kubeadm init and join with the WaitForAllControlPlaneComponents feature gate enabled. +vars: + # vars defines default values for variable used by tasks in this workflow; + # those values might be overridden when importing this files. + initVersion: v1.12.8 + upgradeVersion: v1.13.5 + controlPlaneNodes: 3 + workerNodes: 2 + baseImage: kindest/base:v20221102-76f15095 # has containerd + image: kindest/node:test + clusterName: kinder-wait-for-all-control-plane-components + kubeadmVerbosity: 6 +tasks: +- name: pull-base-image + description: | + pulls kindest/base image with docker in docker and all the prerequisites necessary for running kind(er) + cmd: docker + args: + - pull + - "{{ .vars.baseImage }}" +- name: add-kubernetes-versions + description: | + creates a node-image-variant by adding Kubernetes version "initVersion" + to be used when executing "kinder do kubeadm-init". + cmd: kinder + args: + - build + - node-image-variant + - --base-image={{ .vars.baseImage }} + - --image={{ .vars.image }} + - --with-init-artifacts={{ .vars.initVersion }} + - --loglevel=debug + timeout: 15m +- name: create-cluster + description: | + create a set of nodes ready for hosting the Kubernetes cluster + cmd: kinder + args: + - create + - cluster + - --name={{ .vars.clusterName }} + - --image={{ .vars.image }} + - --control-plane-nodes={{ .vars.controlPlaneNodes }} + - --worker-nodes={{ .vars.workerNodes }} + - --loglevel=debug + timeout: 5m +- name: init + description: | + Initializes the Kubernetes cluster with version "initVersion" + by starting the boostrap control-plane nodes + cmd: kinder + args: + - do + - kubeadm-init + - --name={{ .vars.clusterName }} + - --copy-certs=auto + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + - --kubeadm-feature-gate="WaitForAllControlPlaneComponents=true" + timeout: 5m +- name: join + description: | + Join the other nodes to the Kubernetes cluster + cmd: kinder + args: + - do + - kubeadm-join + - --name={{ .vars.clusterName }} + - --copy-certs=auto + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + timeout: 10m +- name: cluster-info + description: | + Runs cluster-info on the cluster before upgrade + cmd: kinder + args: + - do + - cluster-info + - --name={{ .vars.clusterName }} + - --loglevel=debug +- name: get-logs + description: | + Collects all the test logs + cmd: kinder + args: + - export + - logs + - --loglevel=debug + - --name={{ .vars.clusterName }} + - "{{ .env.ARTIFACTS }}" + force: true + timeout: 5m + # kind export log is know to be flaky, so we are temporary ignoring errors in order + # to make the test pass in case everything else passed + # see https://github.com/kubernetes-sigs/kind/issues/456 + ignoreError: true +- name: reset + description: | + Exec kubeadm reset + cmd: kinder + args: + - do + - kubeadm-reset + - --name={{ .vars.clusterName }} + - --loglevel=debug + - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }} + force: true +- name: delete + description: | + Deletes the cluster + cmd: kinder + args: + - delete + - cluster + - --name={{ .vars.clusterName }} + - --loglevel=debug + force: true