kinder: add e2e test for WaitForAllControlPlaneComponents

kubernetes · Feb 28, 2024 · e87ff14 · e87ff14
1 parent 90085a9
commit e87ff14
Show file tree

Hide file tree

Showing 6 changed files with 318 additions and 0 deletions.
diff --git a/kinder/ci/tools/update-workflows/config.yaml b/kinder/ci/tools/update-workflows/config.yaml
@@ -237,3 +237,15 @@ jobGroups:
     - ./templates/workflows/super-admin-tasks.yaml
   jobs:
   - kubernetesVersion: latest
+
+- name: wait-for-all-control-plane-components
+  testInfraJobSpec:
+    targetFile: kubeadm-kinder-wait-for-all-control-plane-components.yaml
+    template: ./templates/testinfra/kubeadm-kinder-wait-for-all-control-plane-components.yaml
+  kinderWorkflowSpec:
+    targetFile: wait-for-all-control-plane-components-{{ .KubernetesVersion }}.yaml
+    template: ./templates/workflows/wait-for-all-control-plane-components.yaml
+    additionalFiles:
+    - ./templates/workflows/wait-for-all-control-plane-components-tasks.yaml
+  jobs:
+  - kubernetesVersion: latest
diff --git a/...e-workflows/templates/testinfra/kubeadm-kinder-wait-for-all-control-plane-components.yaml b/...e-workflows/templates/testinfra/kubeadm-kinder-wait-for-all-control-plane-components.yaml
@@ -0,0 +1,42 @@
+- name: ci-kubernetes-e2e-kubeadm-kinder-wait-for-all-control-plane-components-{{ dashVer .KubernetesVersion }}
+  cluster: k8s-infra-prow-build
+  interval: {{ .JobInterval }}
+  decorate: true
+  labels:
+    preset-dind-enabled: "true"
+    preset-kind-volume-mounts: "true"
+  annotations:
+    testgrid-dashboards: sig-cluster-lifecycle-kubeadm
+    testgrid-tab-name: kubeadm-kinder-wait-for-all-control-plane-components-{{ dashVer .KubernetesVersion }}
+    testgrid-alert-email: sig-cluster-lifecycle-kubeadm-alerts@kubernetes.io
+    description: "OWNER: sig-cluster-lifecycle (kinder); Uses kubeadm/kinder to create a cluster with the WaitForAllControlPlaneComponents feature gate"
+    testgrid-num-columns-recent: "20"
+{{ .AlertAnnotations }}
+  decoration_config:
+    timeout: 60m
+  extra_refs:
+  - org: kubernetes
+    repo: kubernetes
+    base_ref: {{ branchFor .KubernetesVersion }}
+    path_alias: k8s.io/kubernetes
+  - org: kubernetes
+    repo: kubeadm
+    base_ref: main
+    path_alias: k8s.io/kubeadm
+  spec:
+    containers:
+    - image: gcr.io/k8s-staging-test-infra/kubekins-e2e:{{ .TestInfraImage }}-{{ imageVer .KubernetesVersion }}
+      command:
+      - runner.sh
+      - "../kubeadm/kinder/ci/kinder-run.sh"
+      args:
+      - {{ .WorkflowFile }}
+      securityContext:
+        privileged: true
+      resources:
+        limits:
+          memory: "9000Mi"
+          cpu: 2000m
+        requests:
+          memory: "9000Mi"
+          cpu: 2000m
diff --git a/...ols/update-workflows/templates/workflows/wait-for-all-control-plane-components-tasks.yaml b/...ols/update-workflows/templates/workflows/wait-for-all-control-plane-components-tasks.yaml
@@ -0,0 +1,121 @@
+# IMPORTANT! this workflow is imported by wait-for-all-control-plane-components* workflows.
+version: 1
+summary: |
+  This workflow implements a sequence of tasks used to test kubeadm init and join with the WaitForAllControlPlaneComponents feature gate enabled.
+vars:
+  # vars defines default values for variable used by tasks in this workflow;
+  # those values might be overridden when importing this files.
+  initVersion: v1.12.8
+  upgradeVersion: v1.13.5
+  controlPlaneNodes: 3
+  workerNodes: 2
+  baseImage: kindest/base:v20221102-76f15095 # has containerd
+  image: kindest/node:test
+  clusterName: kinder-wait-for-all-control-plane-components
+  kubeadmVerbosity: 6
+tasks:
+- name: pull-base-image
+  description: |
+    pulls kindest/base image with docker in docker and all the prerequisites necessary for running kind(er)
+  cmd: docker
+  args:
+    - pull
+    - "{{ .vars.baseImage }}"
+- name: add-kubernetes-versions
+  description: |
+    creates a node-image-variant by adding Kubernetes version "initVersion"
+    to be used when executing "kinder do kubeadm-init".
+  cmd: kinder
+  args:
+    - build
+    - node-image-variant
+    - --base-image={{ .vars.baseImage }}
+    - --image={{ .vars.image }}
+    - --with-init-artifacts={{ .vars.initVersion }}
+    - --loglevel=debug
+  timeout: 15m
+- name: create-cluster
+  description: |
+    create a set of nodes ready for hosting the Kubernetes cluster
+  cmd: kinder
+  args:
+    - create
+    - cluster
+    - --name={{ .vars.clusterName }}
+    - --image={{ .vars.image }}
+    - --control-plane-nodes={{ .vars.controlPlaneNodes }}
+    - --worker-nodes={{ .vars.workerNodes }}
+    - --loglevel=debug
+  timeout: 5m
+- name: init
+  description: |
+    Initializes the Kubernetes cluster with version "initVersion"
+    by starting the boostrap control-plane nodes
+  cmd: kinder
+  args:
+    - do
+    - kubeadm-init
+    - --name={{ .vars.clusterName }}
+    - --copy-certs=auto
+    - --loglevel=debug
+    - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
+    - --kubeadm-feature-gate="WaitForAllControlPlaneComponents=true"
+  timeout: 5m
+- name: join
+  description: |
+    Join the other nodes to the Kubernetes cluster
+  cmd: kinder
+  args:
+    - do
+    - kubeadm-join
+    - --name={{ .vars.clusterName }}
+    - --copy-certs=auto
+    - --loglevel=debug
+    - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
+  timeout: 10m
+- name: cluster-info
+  description: |
+    Runs cluster-info on the cluster before upgrade
+  cmd: kinder
+  args:
+    - do
+    - cluster-info
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+- name: get-logs
+  description: |
+    Collects all the test logs
+  cmd: kinder
+  args:
+    - export
+    - logs
+    - --loglevel=debug
+    - --name={{ .vars.clusterName }}
+    - "{{ .env.ARTIFACTS }}"
+  force: true
+  timeout: 5m
+  # kind export log is know to be flaky, so we are temporary ignoring errors in order
+  # to make the test pass in case everything else passed
+  # see https://github.com/kubernetes-sigs/kind/issues/456
+  ignoreError: true
+- name: reset
+  description: |
+    Exec kubeadm reset
+  cmd: kinder
+  args:
+    - do
+    - kubeadm-reset
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+    - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
+  force: true
+- name: delete
+  description: |
+    Deletes the cluster
+  cmd: kinder
+  args:
+    - delete
+    - cluster
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+  force: true
diff --git a/.../ci/tools/update-workflows/templates/workflows/wait-for-all-control-plane-components.yaml b/.../ci/tools/update-workflows/templates/workflows/wait-for-all-control-plane-components.yaml
@@ -0,0 +1,10 @@
+version: 1
+summary: |
+  This workflow tests if kubeadm {{ .InitVersion }} can "init" and "join" nodes with the feature gate
+  WaitForAllControlPlaneComponents feature enabled. The Kubernetes version is at {{ .KubernetesVersion }}.
+  test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-wait-for-all-control-plane-components-{{ dashVer .KubernetesVersion }}
+  config    > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/{{ .TargetFile }}
+vars:
+  initVersion: "\{\{ resolve `ci/{{ ciLabelFor .InitVersion }}` \}\}"
+tasks:
+- import: wait-for-all-control-plane-components-tasks.yaml
diff --git a/kinder/ci/workflows/wait-for-all-control-plane-components-latest.yaml b/kinder/ci/workflows/wait-for-all-control-plane-components-latest.yaml
@@ -0,0 +1,11 @@
+# AUTOGENERATED by https://git.k8s.io/kubeadm/kinder/ci/tools/update-workflows
+version: 1
+summary: |
+  This workflow tests if kubeadm latest can "init" and "join" nodes with the feature gate
+  WaitForAllControlPlaneComponents feature enabled. The Kubernetes version is at latest.
+  test grid > https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm#kubeadm-kinder-wait-for-all-control-plane-components-latest
+  config    > https://git.k8s.io/test-infra/config/jobs/kubernetes/sig-cluster-lifecycle/kubeadm-kinder-wait-for-all-control-plane-components.yaml
+vars:
+  initVersion: "{{ resolve `ci/latest` }}"
+tasks:
+- import: wait-for-all-control-plane-components-tasks.yaml
diff --git a/kinder/ci/workflows/wait-for-all-control-plane-components-tasks.yaml b/kinder/ci/workflows/wait-for-all-control-plane-components-tasks.yaml
@@ -0,0 +1,122 @@
+# AUTOGENERATED by https://git.k8s.io/kubeadm/kinder/ci/tools/update-workflows
+# IMPORTANT! this workflow is imported by wait-for-all-control-plane-components* workflows.
+version: 1
+summary: |
+  This workflow implements a sequence of tasks used to test kubeadm init and join with the WaitForAllControlPlaneComponents feature gate enabled.
+vars:
+  # vars defines default values for variable used by tasks in this workflow;
+  # those values might be overridden when importing this files.
+  initVersion: v1.12.8
+  upgradeVersion: v1.13.5
+  controlPlaneNodes: 3
+  workerNodes: 2
+  baseImage: kindest/base:v20221102-76f15095 # has containerd
+  image: kindest/node:test
+  clusterName: kinder-wait-for-all-control-plane-components
+  kubeadmVerbosity: 6
+tasks:
+- name: pull-base-image
+  description: |
+    pulls kindest/base image with docker in docker and all the prerequisites necessary for running kind(er)
+  cmd: docker
+  args:
+    - pull
+    - "{{ .vars.baseImage }}"
+- name: add-kubernetes-versions
+  description: |
+    creates a node-image-variant by adding Kubernetes version "initVersion"
+    to be used when executing "kinder do kubeadm-init".
+  cmd: kinder
+  args:
+    - build
+    - node-image-variant
+    - --base-image={{ .vars.baseImage }}
+    - --image={{ .vars.image }}
+    - --with-init-artifacts={{ .vars.initVersion }}
+    - --loglevel=debug
+  timeout: 15m
+- name: create-cluster
+  description: |
+    create a set of nodes ready for hosting the Kubernetes cluster
+  cmd: kinder
+  args:
+    - create
+    - cluster
+    - --name={{ .vars.clusterName }}
+    - --image={{ .vars.image }}
+    - --control-plane-nodes={{ .vars.controlPlaneNodes }}
+    - --worker-nodes={{ .vars.workerNodes }}
+    - --loglevel=debug
+  timeout: 5m
+- name: init
+  description: |
+    Initializes the Kubernetes cluster with version "initVersion"
+    by starting the boostrap control-plane nodes
+  cmd: kinder
+  args:
+    - do
+    - kubeadm-init
+    - --name={{ .vars.clusterName }}
+    - --copy-certs=auto
+    - --loglevel=debug
+    - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
+    - --kubeadm-feature-gate="WaitForAllControlPlaneComponents=true"
+  timeout: 5m
+- name: join
+  description: |
+    Join the other nodes to the Kubernetes cluster
+  cmd: kinder
+  args:
+    - do
+    - kubeadm-join
+    - --name={{ .vars.clusterName }}
+    - --copy-certs=auto
+    - --loglevel=debug
+    - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
+  timeout: 10m
+- name: cluster-info
+  description: |
+    Runs cluster-info on the cluster before upgrade
+  cmd: kinder
+  args:
+    - do
+    - cluster-info
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+- name: get-logs
+  description: |
+    Collects all the test logs
+  cmd: kinder
+  args:
+    - export
+    - logs
+    - --loglevel=debug
+    - --name={{ .vars.clusterName }}
+    - "{{ .env.ARTIFACTS }}"
+  force: true
+  timeout: 5m
+  # kind export log is know to be flaky, so we are temporary ignoring errors in order
+  # to make the test pass in case everything else passed
+  # see https://github.com/kubernetes-sigs/kind/issues/456
+  ignoreError: true
+- name: reset
+  description: |
+    Exec kubeadm reset
+  cmd: kinder
+  args:
+    - do
+    - kubeadm-reset
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+    - --kubeadm-verbosity={{ .vars.kubeadmVerbosity }}
+  force: true
+- name: delete
+  description: |
+    Deletes the cluster
+  cmd: kinder
+  args:
+    - delete
+    - cluster
+    - --name={{ .vars.clusterName }}
+    - --loglevel=debug
+  force: true