diff --git a/.prow.yaml b/.prow.yaml index 8982af152..bf2f3ecc0 100644 --- a/.prow.yaml +++ b/.prow.yaml @@ -25,7 +25,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.15.0 + - image: golang:1.15.7 command: - make args: @@ -67,7 +67,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.15.0 + - image: golang:1.15.7 command: - make args: @@ -88,7 +88,7 @@ presubmits: preset-goproxy: "true" spec: containers: - - image: golang:1.15.0 + - image: golang:1.15.7 command: - make args: @@ -122,10 +122,10 @@ presubmits: memory: 2Gi ######################################################### - # E2E/Conformance tests (AWS, 1.16-1.19) + # E2E/Conformance tests (AWS, 1.18-1.19) ######################################################### - - name: pull-kubeone-e2e-aws-conformance-1.16 + - name: pull-kubeone-e2e-aws-containerd-conformance-1.18 run_if_changed: "(pkg/|examples/terraform/aws|hack/|test/)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -134,33 +134,7 @@ presubmits: preset-aws: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "aws" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.16.15" - - name: TEST_SET - value: "conformance" - resources: - requests: - cpu: 1 - - - name: pull-kubeone-e2e-aws-containerd-conformance-1.16 - run_if_changed: "(pkg/|examples/terraform/aws|hack/|test/)" - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-aws: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -172,14 +146,14 @@ presubmits: - name: CONTAINER_RUNTIME value: containerd - name: TEST_CLUSTER_TARGET_VERSION - value: "1.16.15" + value: "1.18.15" - name: TEST_SET value: "conformance" resources: requests: cpu: 1 - - name: pull-kubeone-e2e-aws-conformance-1.17 + - name: pull-kubeone-e2e-aws-conformance-1.18 run_if_changed: "(pkg/|examples/terraform/aws|hack/|test/)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -188,7 +162,7 @@ presubmits: preset-aws: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -198,14 +172,14 @@ presubmits: - name: PROVIDER value: "aws" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_SET value: "conformance" resources: requests: cpu: 1 - - name: pull-kubeone-e2e-aws-conformance-1.18 + - name: pull-kubeone-e2e-aws-conformance-1.19 run_if_changed: "(pkg/|examples/terraform/aws|hack/|test/)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -214,7 +188,7 @@ presubmits: preset-aws: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -224,14 +198,14 @@ presubmits: - name: PROVIDER value: "aws" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.19.7" - name: TEST_SET value: "conformance" resources: requests: cpu: 1 - - - name: pull-kubeone-e2e-aws-conformance-1.19 + + - name: pull-kubeone-e2e-aws-conformance-1.20 run_if_changed: "(pkg/|examples/terraform/aws|hack/|test/)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -240,7 +214,7 @@ presubmits: preset-aws: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -250,7 +224,7 @@ presubmits: - name: PROVIDER value: "aws" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.20.2" - name: TEST_SET value: "conformance" resources: @@ -258,36 +232,10 @@ presubmits: cpu: 1 ######################################################### - # E2E/Conformance tests (DigitalOcean, 1.16-1.19) + # E2E/Conformance tests (DigitalOcean, 1.18-1.19) ######################################################### - - name: pull-kubeone-e2e-digitalocean-conformance-1.16 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-digitalocean: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "digitalocean" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.16.15" - - name: TEST_SET - value: "conformance" - resources: - requests: - cpu: 1 - - - name: pull-kubeone-e2e-digitalocean-conformance-1.17 + - name: pull-kubeone-e2e-digitalocean-conformance-1.18 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -296,7 +244,7 @@ presubmits: preset-digitalocean: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -306,14 +254,14 @@ presubmits: - name: PROVIDER value: "digitalocean" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_SET value: "conformance" resources: requests: cpu: 1 - - name: pull-kubeone-e2e-digitalocean-conformance-1.18 + - name: pull-kubeone-e2e-digitalocean-conformance-1.19 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -322,7 +270,7 @@ presubmits: preset-digitalocean: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -332,14 +280,14 @@ presubmits: - name: PROVIDER value: "digitalocean" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.19.7" - name: TEST_SET value: "conformance" resources: requests: cpu: 1 - - name: pull-kubeone-e2e-digitalocean-conformance-1.19 + - name: pull-kubeone-e2e-digitalocean-conformance-1.20 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -348,7 +296,7 @@ presubmits: preset-digitalocean: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -358,7 +306,7 @@ presubmits: - name: PROVIDER value: "digitalocean" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.20.2" - name: TEST_SET value: "conformance" resources: @@ -366,36 +314,10 @@ presubmits: cpu: 1 ######################################################### - # E2E/Conformance tests (Hetzner, 1.16-1.19) + # E2E/Conformance tests (Hetzner, 1.18-1.19) ######################################################### - - name: pull-kubeone-e2e-hetzner-conformance-1.16 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-hetzner: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "hetzner" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.16.15" - - name: TEST_SET - value: "conformance" - resources: - requests: - cpu: 1 - - - name: pull-kubeone-e2e-hetzner-conformance-1.17 + - name: pull-kubeone-e2e-hetzner-conformance-1.18 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -404,7 +326,7 @@ presubmits: preset-hetzner: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -414,14 +336,14 @@ presubmits: - name: PROVIDER value: "hetzner" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_SET value: "conformance" resources: requests: cpu: 1 - - name: pull-kubeone-e2e-hetzner-conformance-1.18 + - name: pull-kubeone-e2e-hetzner-conformance-1.19 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -430,7 +352,7 @@ presubmits: preset-hetzner: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -440,14 +362,14 @@ presubmits: - name: PROVIDER value: "hetzner" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.19.7" - name: TEST_SET value: "conformance" resources: requests: cpu: 1 - - name: pull-kubeone-e2e-hetzner-conformance-1.19 + - name: pull-kubeone-e2e-hetzner-conformance-1.20 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -456,7 +378,7 @@ presubmits: preset-hetzner: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -466,7 +388,7 @@ presubmits: - name: PROVIDER value: "hetzner" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.20.2" - name: TEST_SET value: "conformance" resources: @@ -474,38 +396,10 @@ presubmits: cpu: 1 ######################################################### - # E2E/Conformance tests (GCE, 1.16-1.19) + # E2E/Conformance tests (GCE, 1.18-1.19) ######################################################### - - name: pull-kubeone-e2e-gce-conformance-1.16 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-gce: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "gce" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.16.15" - - name: TEST_SET - value: "conformance" - - name: TF_VAR_project - value: "kubeone-terraform-test" - resources: - requests: - cpu: 1 - - - name: pull-kubeone-e2e-gce-conformance-1.17 + - name: pull-kubeone-e2e-gce-conformance-1.18 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -514,7 +408,7 @@ presubmits: preset-gce: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -524,7 +418,7 @@ presubmits: - name: PROVIDER value: "gce" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_SET value: "conformance" - name: TF_VAR_project @@ -533,7 +427,7 @@ presubmits: requests: cpu: 1 - - name: pull-kubeone-e2e-gce-conformance-1.18 + - name: pull-kubeone-e2e-gce-conformance-1.19 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -542,7 +436,7 @@ presubmits: preset-gce: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -552,7 +446,7 @@ presubmits: - name: PROVIDER value: "gce" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.19.7" - name: TEST_SET value: "conformance" - name: TF_VAR_project @@ -561,7 +455,7 @@ presubmits: requests: cpu: 1 - - name: pull-kubeone-e2e-gce-conformance-1.19 + - name: pull-kubeone-e2e-gce-conformance-1.20 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -570,7 +464,7 @@ presubmits: preset-gce: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -580,7 +474,7 @@ presubmits: - name: PROVIDER value: "gce" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.20.2" - name: TEST_SET value: "conformance" - name: TF_VAR_project @@ -593,33 +487,7 @@ presubmits: # E2E/Conformance tests (Packet, 1.16-1.19) ######################################################### - - name: pull-kubeone-e2e-packet-conformance-1.16 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-packet: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "packet" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.16.15" - - name: TEST_SET - value: "conformance" - resources: - requests: - cpu: 1 - - - name: pull-kubeone-e2e-packet-conformance-1.17 + - name: pull-kubeone-e2e-packet-conformance-1.18 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -628,7 +496,7 @@ presubmits: preset-packet: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -638,14 +506,14 @@ presubmits: - name: PROVIDER value: "packet" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_SET value: "conformance" resources: requests: cpu: 1 - - name: pull-kubeone-e2e-packet-conformance-1.18 + - name: pull-kubeone-e2e-packet-conformance-1.19 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -654,7 +522,7 @@ presubmits: preset-packet: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -664,14 +532,14 @@ presubmits: - name: PROVIDER value: "packet" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.19.7" - name: TEST_SET value: "conformance" resources: requests: cpu: 1 - - - name: pull-kubeone-e2e-packet-conformance-1.19 + + - name: pull-kubeone-e2e-packet-conformance-1.20 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -680,7 +548,7 @@ presubmits: preset-packet: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -690,7 +558,7 @@ presubmits: - name: PROVIDER value: "packet" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.20.2" - name: TEST_SET value: "conformance" resources: @@ -698,36 +566,10 @@ presubmits: cpu: 1 ######################################################### - # E2E/Conformance tests (OpenStack, 1.16-1.19) + # E2E/Conformance tests (OpenStack, 1.18-1.19) ######################################################### - - name: pull-kubeone-e2e-openstack-conformance-1.16 - always_run: false - decorate: true - clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" - labels: - preset-goproxy: "true" - preset-openstack: "true" - spec: - containers: - - image: kubermatic/kubeone-e2e:v0.1.12 - imagePullPolicy: Always - command: - - make - args: - - e2e-test - env: - - name: PROVIDER - value: "openstack" - - name: TEST_CLUSTER_TARGET_VERSION - value: "1.16.15" - - name: TEST_SET - value: "conformance" - resources: - requests: - cpu: 1 - - - name: pull-kubeone-e2e-openstack-conformance-1.17 + - name: pull-kubeone-e2e-openstack-conformance-1.18 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -736,7 +578,7 @@ presubmits: preset-openstack: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -746,14 +588,14 @@ presubmits: - name: PROVIDER value: "openstack" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_SET value: "conformance" resources: requests: cpu: 1 - - name: pull-kubeone-e2e-openstack-conformance-1.18 + - name: pull-kubeone-e2e-openstack-conformance-1.19 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -762,7 +604,7 @@ presubmits: preset-openstack: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -772,14 +614,14 @@ presubmits: - name: PROVIDER value: "openstack" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.19.7" - name: TEST_SET value: "conformance" resources: requests: cpu: 1 - - - name: pull-kubeone-e2e-openstack-conformance-1.19 + + - name: pull-kubeone-e2e-openstack-conformance-1.20 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -788,7 +630,7 @@ presubmits: preset-openstack: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -798,7 +640,7 @@ presubmits: - name: PROVIDER value: "openstack" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.20.2" - name: TEST_SET value: "conformance" resources: @@ -809,7 +651,7 @@ presubmits: # E2E/Upgrade tests (AWS) ######################################################### - - name: pull-kubeone-e2e-aws-upgrade-1.16-1.17 + - name: pull-kubeone-e2e-aws-upgrade-containerd-1.17-1.18 run_if_changed: "(pkg/|examples/terraform/aws|hack/|test/)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -818,7 +660,7 @@ presubmits: preset-aws: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -827,14 +669,16 @@ presubmits: env: - name: PROVIDER value: "aws" + - name: CONTAINER_RUNTIME + value: containerd - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.16.15" + value: "1.17.17" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_SET value: "upgrades" - - name: pull-kubeone-e2e-aws-upgrade-containerd-1.16-1.17 + - name: pull-kubeone-e2e-aws-upgrade-1.17-1.18 run_if_changed: "(pkg/|examples/terraform/aws|hack/|test/)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -843,7 +687,7 @@ presubmits: preset-aws: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -852,16 +696,14 @@ presubmits: env: - name: PROVIDER value: "aws" - - name: CONTAINER_RUNTIME - value: containerd - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.16.15" + value: "1.17.17" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_SET value: "upgrades" - - name: pull-kubeone-e2e-aws-upgrade-1.17-1.18 + - name: pull-kubeone-e2e-aws-upgrade-1.18-1.19 run_if_changed: "(pkg/|examples/terraform/aws|hack/|test/)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -870,7 +712,7 @@ presubmits: preset-aws: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -880,13 +722,13 @@ presubmits: - name: PROVIDER value: "aws" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.19.7" - name: TEST_SET value: "upgrades" - - - name: pull-kubeone-e2e-aws-upgrade-1.18-1.19 + + - name: pull-kubeone-e2e-aws-upgrade-1.19-1.20 run_if_changed: "(pkg/|examples/terraform/aws|hack/|test/)" decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -895,7 +737,7 @@ presubmits: preset-aws: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -905,9 +747,9 @@ presubmits: - name: PROVIDER value: "aws" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.18.9" + value: "1.19.7" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.20.2" - name: TEST_SET value: "upgrades" @@ -915,7 +757,7 @@ presubmits: # E2E/Upgrade tests (DigitalOcean) ######################################################### - - name: pull-kubeone-e2e-digitalocean-upgrade-1.16-1.17 + - name: pull-kubeone-e2e-digitalocean-upgrade-1.17-1.18 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -924,7 +766,7 @@ presubmits: preset-digitalocean: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -934,13 +776,13 @@ presubmits: - name: PROVIDER value: "digitalocean" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.16.15" + value: "1.17.17" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_SET value: "upgrades" - - name: pull-kubeone-e2e-digitalocean-upgrade-1.17-1.18 + - name: pull-kubeone-e2e-digitalocean-upgrade-1.18-1.19 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -949,7 +791,7 @@ presubmits: preset-digitalocean: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -959,13 +801,13 @@ presubmits: - name: PROVIDER value: "digitalocean" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.19.7" - name: TEST_SET value: "upgrades" - - - name: pull-kubeone-e2e-digitalocean-upgrade-1.18-1.19 + + - name: pull-kubeone-e2e-digitalocean-upgrade-1.19-1.20 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -974,7 +816,7 @@ presubmits: preset-digitalocean: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -984,9 +826,9 @@ presubmits: - name: PROVIDER value: "digitalocean" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.18.9" + value: "1.19.7" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.20.2" - name: TEST_SET value: "upgrades" @@ -994,7 +836,7 @@ presubmits: # E2E/Upgrade tests (Hetzner) ######################################################### - - name: pull-kubeone-e2e-hetzner-upgrade-1.16-1.17 + - name: pull-kubeone-e2e-hetzner-upgrade-1.17-1.18 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -1003,7 +845,7 @@ presubmits: preset-hetzner: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -1013,13 +855,13 @@ presubmits: - name: PROVIDER value: "hetzner" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.16.15" + value: "1.17.17" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_SET value: "upgrades" - - name: pull-kubeone-e2e-hetzner-upgrade-1.17-1.18 + - name: pull-kubeone-e2e-hetzner-upgrade-1.18-1.19 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -1028,7 +870,7 @@ presubmits: preset-hetzner: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -1038,13 +880,13 @@ presubmits: - name: PROVIDER value: "hetzner" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.19.7" - name: TEST_SET value: "upgrades" - - - name: pull-kubeone-e2e-hetzner-upgrade-1.18-1.19 + + - name: pull-kubeone-e2e-hetzner-upgrade-1.19-1.20 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -1053,7 +895,7 @@ presubmits: preset-hetzner: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -1063,9 +905,9 @@ presubmits: - name: PROVIDER value: "hetzner" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.18.9" + value: "1.19.7" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.20.2" - name: TEST_SET value: "upgrades" @@ -1073,7 +915,7 @@ presubmits: # E2E/Upgrade tests (GCE) ######################################################### - - name: pull-kubeone-e2e-gce-upgrade-1.16-1.17 + - name: pull-kubeone-e2e-gce-upgrade-1.17-1.18 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -1082,7 +924,7 @@ presubmits: preset-gce: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -1092,15 +934,15 @@ presubmits: - name: PROVIDER value: "gce" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.16.15" + value: "1.17.17" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_SET value: "upgrades" - name: TF_VAR_project value: "kubeone-terraform-test" - - name: pull-kubeone-e2e-gce-upgrade-1.17-1.18 + - name: pull-kubeone-e2e-gce-upgrade-1.18-1.19 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -1109,7 +951,7 @@ presubmits: preset-gce: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -1119,15 +961,15 @@ presubmits: - name: PROVIDER value: "gce" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.19.7" - name: TEST_SET value: "upgrades" - name: TF_VAR_project value: "kubeone-terraform-test" - - - name: pull-kubeone-e2e-gce-upgrade-1.18-1.19 + + - name: pull-kubeone-e2e-gce-upgrade-1.19-1.20 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -1136,7 +978,7 @@ presubmits: preset-gce: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -1146,9 +988,9 @@ presubmits: - name: PROVIDER value: "gce" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.18.9" + value: "1.19.7" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.20.2" - name: TEST_SET value: "upgrades" - name: TF_VAR_project @@ -1158,7 +1000,7 @@ presubmits: # E2E/Upgrade tests (Packet) ######################################################### - - name: pull-kubeone-e2e-packet-upgrade-1.16-1.17 + - name: pull-kubeone-e2e-packet-upgrade-1.17-1.18 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -1167,7 +1009,7 @@ presubmits: preset-packet: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -1177,13 +1019,13 @@ presubmits: - name: PROVIDER value: "packet" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.16.15" + value: "1.17.17" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_SET value: "upgrades" - - name: pull-kubeone-e2e-packet-upgrade-1.17-1.18 + - name: pull-kubeone-e2e-packet-upgrade-1.18-1.19 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -1192,7 +1034,7 @@ presubmits: preset-packet: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -1202,13 +1044,13 @@ presubmits: - name: PROVIDER value: "packet" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.19.7" - name: TEST_SET value: "upgrades" - - - name: pull-kubeone-e2e-packet-upgrade-1.18-1.19 + + - name: pull-kubeone-e2e-packet-upgrade-1.19-1.20 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -1217,7 +1059,7 @@ presubmits: preset-packet: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -1227,9 +1069,9 @@ presubmits: - name: PROVIDER value: "packet" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.18.9" + value: "1.19.7" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.20.2" - name: TEST_SET value: "upgrades" @@ -1237,7 +1079,7 @@ presubmits: # E2E/Upgrade tests (OpenStack) ######################################################### - - name: pull-kubeone-e2e-openstack-upgrade-1.16-1.17 + - name: pull-kubeone-e2e-openstack-upgrade-1.17-1.18 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -1246,7 +1088,7 @@ presubmits: preset-openstack: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -1256,13 +1098,13 @@ presubmits: - name: PROVIDER value: "openstack" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.16.15" + value: "1.17.17" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_SET value: "upgrades" - - name: pull-kubeone-e2e-openstack-upgrade-1.17-1.18 + - name: pull-kubeone-e2e-openstack-upgrade-1.18-1.19 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -1271,7 +1113,7 @@ presubmits: preset-openstack: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -1281,13 +1123,13 @@ presubmits: - name: PROVIDER value: "openstack" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.17.12" + value: "1.18.15" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.18.9" + value: "1.19.7" - name: TEST_SET value: "upgrades" - - - name: pull-kubeone-e2e-openstack-upgrade-1.18-1.19 + + - name: pull-kubeone-e2e-openstack-upgrade-1.19-1.20 always_run: false decorate: true clone_uri: "ssh://git@github.com/kubermatic/kubeone.git" @@ -1296,7 +1138,7 @@ presubmits: preset-openstack: "true" spec: containers: - - image: kubermatic/kubeone-e2e:v0.1.12 + - image: kubermatic/kubeone-e2e:v0.1.13 imagePullPolicy: Always command: - make @@ -1306,9 +1148,9 @@ presubmits: - name: PROVIDER value: "openstack" - name: TEST_CLUSTER_INITIAL_VERSION - value: "1.18.9" + value: "1.19.7" - name: TEST_CLUSTER_TARGET_VERSION - value: "1.19.2" + value: "1.20.2" - name: TEST_SET value: "upgrades" diff --git a/CHANGELOG.md b/CHANGELOG.md index 115e861f2..4a50609dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,124 @@ # Changelog +# [v1.2.0-beta.1](https://github.com/kubermatic/kubeone/releases/tag/v1.2.0-beta.1) - 2021-02-17 + +## Attention Needed + +* [**Breaking**] Support for CoreOS has been removed from KubeOne and machine-controller + * CoreOS has reached End-of-Life on May 26, 2020 + * As an alternative to CoreOS, KubeOne supports Flatcar Linux + * We recommend migrating your CoreOS clusters to the Flatcar Linux or other supported operating system +* [**Breaking**] Default values for OpenIDConnect has been corrected to match what's advised by the example configuration + * Previously, there were no default values for the OpenIDConnect fields + * This might only affect users using the OpenIDConnect feature +* [**Breaking**] Disallow and deprecate the PodPresets feature + * [**Action Required**] If you're upgrading a cluster that uses the PodPresets feature from Kubernetes 1.19 to 1.20, you have to disable the PodPresets feature in the KubeOne configuration manifest + * The PodPresets feature has been removed from Kubernetes 1.20 with no built-in replacement + * It's not possible to use the PodPresets feature starting with Kubernetes 1.20, however, it currently remains possible to use it for older Kubernetes versions + * The PodPresets feature will be removed from the KubeOneCluster API once Kubernetes 1.19 reaches End-of-Life (EOL) + * As an alternative to the PodPresets feature, Kubernetes recommends using the MutatingAdmissionWebhooks. + +## Added + +* Add support for Kubernetes 1.20 + * Previously, we've shared that there is an issue affecting newly created clusters where the first control plane node is unhealthy/broken for the first 5-10 minutes. We've investigated the issue and found out that the issue can be successfully mitigated by restarting the first API server. We've implemented a task that automatically restarts the API server if it's affected by the issue ([#1243](https://github.com/kubermatic/kubeone/pull/1243), [#1245](https://github.com/kubermatic/kubeone/pull/1245)) +* Add support for Debian on control plane and static worker nodes ([#1233](https://github.com/kubermatic/kubeone/pull/1233)) + * Debian is currently not supported by machine-controller, so it's not possible to use it on worker nodes managed by machine-controller + +## Changed + +### API Changes + +* [**Breaking**] Default values for OpenIDConnect has been corrected to match what's advised by the example configuration ([#1235](https://github.com/kubermatic/kubeone/pull/1235)) + * Previously, there were no default values for the OpenIDConnect fields + * This might only affect users using the OpenIDConnect feature +* [**Breaking**] Disallow and deprecate the PodPresets feature ([#1236](https://github.com/kubermatic/kubeone/pull/1236)) + * [**Action Required**] If you're upgrading a cluster that uses the PodPresets feature from Kubernetes 1.19 to 1.20, you have to disable the PodPresets feature in the KubeOne configuration manifest + * The PodPresets feature has been removed from Kubernetes 1.20 with no built-in replacement + * It's not possible to use the PodPresets feature starting with Kubernetes 1.20, however, it currently remains possible to use it for older Kubernetes versions + * The PodPresets feature will be removed from the KubeOneCluster API once Kubernetes 1.19 reaches End-of-Life (EOL) + * As an alternative to the PodPresets feature, Kubernetes recommends using the MutatingAdmissionWebhooks. + +### General + +* Add rsync on CentOS and Amazon Linux ([#1240](https://github.com/kubermatic/kubeone/pull/1240)) + +### Bug Fixes + +* Drop mounting Flexvolume plugins into the OpenStack CCM. This fixes the issue with deploying the OpenStack CCM on the clusters running Flatcar Linux ([#1234](https://github.com/kubermatic/kubeone/pull/1234)) +* Ensure all credentials are available to be used in addons. This fixes the issue with the Backups addon not working on non-AWS providers ([#1248](https://github.com/kubermatic/kubeone/pull/1248)) + +### Updated + +* Update machine-controller to v1.25.0 ([#1238](https://github.com/kubermatic/kubeone/pull/1238)) + +## Removed + +* [**Breaking**] Support for CoreOS has been removed from KubeOne and machine-controller ([#1232](https://github.com/kubermatic/kubeone/pull/1232)) + * CoreOS has reached End-of-Life on May 26, 2020 + * As an alternative to CoreOS, KubeOne supports Flatcar Linux + * We recommend migrating your CoreOS clusters to the Flatcar Linux or other supported operating system + +# [v1.2.0-beta.0](https://github.com/kubermatic/kubeone/releases/tag/v1.2.0-beta.0) - 2021-01-27 + +## Attention Needed + +* Kubernetes has announced deprecation of the Docker (dockershim) support in + the Kubernetes 1.20 release. It's expected that Docker support will be + removed in Kubernetes 1.22 + * All newly created clusters running Kubernetes 1.21+ will be provisioned + with containerd instead of Docker + * Automated migration from Docker to containerd is currently not available, + but is planned for one of the upcoming KubeOne releases + * We highly recommend using containerd instead of Docker for all newly + created clusters. You can opt-in to use containerd instead of Docker by + adding `containerRuntime` configuration to your KubeOne configuration + manifest: + ```yaml + containerRuntime: + containerd: {} + ``` + For the configuration file reference, run `kubeone config print --full`. + + +## Known Issues + +* Provisioning Kubernetes 1.20 clusters results with one of the control plane + nodes being unhealthy/broken for the first 5-10 minutes after provisioning + the cluster. This causes KubeOne to fail to create MachineDeployment objects + because the `machine-controller-webhook` service can't be found. Also, one of + the NodeLocalDNS pods might get stuck in the crash loop. + * KubeOne currently still doesn't support Kubernetes 1.20. We do **not** + recommend provisioning 1.20 clusters or upgrading existing clusters to + Kubernetes 1.20 + * We're currently investigating the issue. You can follow the progress + in the issue [#1222](https://github.com/kubermatic/kubeone/issues/1222) + +## Added + +* Add support for containerd container runtime ([#1180](https://github.com/kubermatic/kubeone/pull/1180), [#1188](https://github.com/kubermatic/kubeone/pull/1188), [#1190](https://github.com/kubermatic/kubeone/pull/1190), [#1205](https://github.com/kubermatic/kubeone/pull/1205), [#1227](https://github.com/kubermatic/kubeone/pull/1227), [#1229](https://github.com/kubermatic/kubeone/pull/1229)) + * Kubernetes has announced deprecation of the Docker (dockershim) support in + the Kubernetes 1.20 release. It's expected that Docker support will be + removed in Kubernetes 1.22 + * All newly created clusters running Kubernetes 1.21+ will default to + containerd instead of Docker + * Automated migration from Docker to containerd is currently not available, + but is planned for one of the upcoming KubeOne releases + +## Changed + +### Bug Fixes + +* Fix wrong legacy Docker version on RPM systems ([#1191](https://github.com/kubermatic/kubeone/pull/1191)) + +### Terraform Configs + +* Replace GoBetween load-balancer in vSphere Terraform example by keepalived ([#1217](https://github.com/kubermatic/kubeone/pull/1217)) + +### Addons + +* Fix DNS resolution issues for the Backups addon ([#1179](https://github.com/kubermatic/kubeone/pull/1179)) + # [v1.2.0-alpha.0](https://github.com/kubermatic/kubeone/releases/tag/v1.2.0-alpha.0) - 2020-11-27 ## Added diff --git a/README.md b/README.md index 7a2679a53..5d0ca5cd2 100644 --- a/README.md +++ b/README.md @@ -8,16 +8,9 @@ Kubermatic KubeOne automates cluster operations on all your cloud, on-prem, edge, and IoT environments. KubeOne can install high-available (HA) master clusters as well single master clusters. -## KubeOne User Survey - -**We're organizing the [KubeOne User Survey][survey]!** -This survey is intended to shape the future roadmap of KubeOne. Your answers -will help us determine future features and schedules. We’re raffling one 10€ -Amazon gift card among the respondents of our KubeOne Survey. - ## Getting Started -All user documentation is available at the +All user documentation for the latest stable version is available at the [KubeOne docs website][docs]. Information about the support policy (natively-supported providers, supported @@ -40,8 +33,9 @@ curl -sfL get.kubeone.io | sh ``` The installation script downloads the release archive from GitHub, installs the -KubeOne binary in your `/usr/local/bin` directory and unpacks the example -Terraform configs in your current working directory. +KubeOne binary in your `/usr/local/bin` directory, and unpacks the example +Terraform configs, addons, and helper scripts in your current working +directory. For other installation methods, check the [Getting KubeOne guide][docs-install] on our documentation website. @@ -118,12 +112,12 @@ See [the list of releases][changelog] to find out about feature changes. [upstream-supported-versions]: https://kubernetes.io/docs/setup/release/version-skew-policy/#supported-versions [cluster-api]: https://github.com/kubernetes-sigs/cluster-api [machine-controller]: https://github.com/kubermatic/machine-controller -[docs]: https://docs.kubermatic.com/kubeone/master/ -[docs-compatibility]: https://docs.kubermatic.com/kubeone/master/compatibility_info/ -[docs-prerequisistes]: https://docs.kubermatic.com/kubeone/master/prerequisites/ -[docs-infrastructure]: https://docs.kubermatic.com/kubeone/master/infrastructure/ -[docs-provisioning]: https://docs.kubermatic.com/kubeone/master/provisioning/ -[docs-install]: https://docs.kubermatic.com/kubeone/master/getting_kubeone/ +[docs]: https://docs.kubermatic.com/kubeone/ +[docs-compatibility]: https://docs.kubermatic.com/kubeone/v1.2/compatibility_info/ +[docs-prerequisistes]: https://docs.kubermatic.com/kubeone/v1.2/prerequisites/ +[docs-infrastructure]: https://docs.kubermatic.com/kubeone/v1.2/infrastructure/ +[docs-provisioning]: https://docs.kubermatic.com/kubeone/v1.2/provisioning/ +[docs-install]: https://docs.kubermatic.com/kubeone/v1.2/getting_kubeone/ [contributing-guide]: https://github.com/kubermatic/KubeOne/blob/master/CONTRIBUTING.md [k8s-slack-kubeone]: https://kubernetes.slack.com/messages/CNEV2UMT7 [k8s-slack]: http://slack.k8s.io/ diff --git a/docs/proposals/20210112-encryption-roviders.md b/docs/proposals/20210112-encryption-roviders.md new file mode 100644 index 000000000..eb4ac7c2b --- /dev/null +++ b/docs/proposals/20210112-encryption-roviders.md @@ -0,0 +1,115 @@ +# Encryption Providers for encrypted secrets at rest + +**Auther**: Mohamed Elsayed (@moelsayed) +**Status**: Draft + + +## Abstract + +By default, all Kubernetes secret objects are stored on disk in plain text inside etcd. The [Encryption Providers](https://kubernetes.io/docs/tasks/administer-cluster/encrypt-data/ +) feature was added to Kubernets starting with version 1.13. + +At rest data encryption is a requirement for security compliance and adds an additional layer of security for secret data, especially when etcd nodes are separated from the control plan and in off-node backups. + +KubeOne needs to support this feature natively. Meaning the user should be able to enable, disable the feature and rotate keys when needed without having to apply any actions manually. + +## Goals + +* Provide a safe path to enable/disable Encryption Providers. +* Support atomic(?) rotation for existing keys. +* Rewriting all secret resources (no just secrets) after enable/disable/rotate operations. + +## Non-Goals + +* Deploy External KMS. +* Safely manage (disable/enable/rotate) configuration when a custom configuration file is used. + +## Challenges + +The feature has a lot of moving parts; as it requires performing a specific sequence of actions, including changing the KubeAPI configuration, restarting KubeAPI and rewriting all secret resources to apply the encryption. This requires the implementation to be as idempotent as possible with ability to rollback on failure, with out breaking the cluster. + +## Implementation + +Unfortunately, it's not possible to simply update the KubeAPI configuration and expect the configuration to reconcile. KubeOne will have to _read_ the _current_ configuration on the cluster, _mutate_ it based on the _required_ state and then apply it. Additionally, KubeOne will have to be able to revert changes on any errors and recover safely if the process is interrupted at any point. + +The configuration for this will be added under `features` in the KubeOneCluster spec: + +```yaml +apiVersion: kubeone.io/v1beta1 +kind: KubeOneCluster +features: + encryptionProviders: + enabled: true + customProvidersFile: | + apiVersion: apiserver.config.k8s.io/v1 + kind: EncryptionConfiguration + resources: + - resources: + - secrets + providers: + - identity: {} + - aescbc: + keys: + - name: key1 + secret: +``` + +To allow users to rotate the keys, a new flag will be added to the `apply` command: + +```bash +--rotate-encryption-key automatically rotate encryption provider key +``` + +### pre-flight checks + + * Cluster is healthy. + * Current Encryption Providers state/configuration is valid and identical on all control plane nodes. + +### Enable Encryption Providers for new cluster + +* Generate a valid configuration file with the `identity` provider set last. +* Sync the configuration file to all Control Plane nodes. +* Set the required KubeAPI configuration and deploy KubeAPI. + +### Enable Encryption Providers for existing cluster + +* Ensure there is no Encryption Provider Config (manually added by the user, broken previous enable process, etc..) present. +* Generate a valid configuration file with the `identity` provider set last. +* Sync the configuration file to all Control Plane nodes. +* Update and restart KubeAPI on all nodes. +* Rewrite secrets to ensure they are encrypted successfully. + +### Disable Encryption Providers for existing cluster + +* Read the current active Encryption Provider configuration from control plane nodes. +* Mutate the configuration to add `identity` provider first and the active provider last. +* Sync the configuration file to all Control Plane nodes. +* Restart KubeAPI on all control plane nodes. +* Rewrite secrets to ensure they are decrypted successfully. +* Update KubeAPI configuration to remove the Encryption Provider configuration and restart KubeAPI on all control plane nodes. +* Remove the old configuration file from all control plane nodes. + +### Rotate Encryption Provider Key for existing cluster + +* Read the current active Encryption Provider configuration from control plane nodes. +* Generate a new encryption key. +* Mutate the configuration file to include the new key first, current key second and `identity` last. +* Sync the updated configuration file to all control plane nodes and restart KubeAPI. +* Rewrite all secrets to ensure they are encrypted with the new key. +* Mutate the configuration file again to remove the old key. +* Sync the updated configuration file to all control plane nodes and restart KubeAPI. + +### Apply Custom Encryption Provider file +This use case is useful for users who would like to utilize an external KMS provider or specify resources other than secrets for encryption. In this case, KubeOne will not manage the content of the file, it will only validate it to make sure it's syntactically valid. Additionally, KubeOne will not rewrite the resources in this case. + +* Ensure the configuration file is valid. +* Sync the configuration file to all control plane nodes. +* Restart KubeAPI on all nodes. + +## Tasks & effort + +* Implement the needed pre-flight checks. +* Implement validation for Encryption Provider configuration files. +* Implement the workflow for each use case. +* Add e2e tests for each workflow. +* Add documentation for the feature. \ No newline at end of file diff --git a/examples/terraform/azure/variables.tf b/examples/terraform/azure/variables.tf index ee3b63694..6a685de56 100644 --- a/examples/terraform/azure/variables.tf +++ b/examples/terraform/azure/variables.tf @@ -24,7 +24,6 @@ variable "worker_os" { # valid choices are: # * ubuntu # * centos - # * coreos default = "ubuntu" } diff --git a/examples/terraform/digitalocean/variables.tf b/examples/terraform/digitalocean/variables.tf index a321fa5d8..8b7318130 100644 --- a/examples/terraform/digitalocean/variables.tf +++ b/examples/terraform/digitalocean/variables.tf @@ -24,7 +24,6 @@ variable "worker_os" { # valid choices are: # * ubuntu # * centos - # * coreos default = "ubuntu" } diff --git a/examples/terraform/gce/variables.tf b/examples/terraform/gce/variables.tf index db4dddcb7..b6ebfd1b1 100644 --- a/examples/terraform/gce/variables.tf +++ b/examples/terraform/gce/variables.tf @@ -24,7 +24,6 @@ variable "worker_os" { # valid choices are: # * ubuntu # * centos - # * coreos default = "ubuntu" } diff --git a/examples/terraform/hetzner/variables.tf b/examples/terraform/hetzner/variables.tf index 261d3904b..c8bf98477 100644 --- a/examples/terraform/hetzner/variables.tf +++ b/examples/terraform/hetzner/variables.tf @@ -24,7 +24,6 @@ variable "worker_os" { # valid choices are: # * ubuntu # * centos - # * coreos default = "ubuntu" } diff --git a/examples/terraform/openstack/variables.tf b/examples/terraform/openstack/variables.tf index 4b7c31f43..2a3f0d5a9 100644 --- a/examples/terraform/openstack/variables.tf +++ b/examples/terraform/openstack/variables.tf @@ -24,7 +24,6 @@ variable "worker_os" { # valid choices are: # * ubuntu # * centos - # * coreos default = "ubuntu" } diff --git a/examples/terraform/packet/variables.tf b/examples/terraform/packet/variables.tf index 7a89f9c32..18b575466 100644 --- a/examples/terraform/packet/variables.tf +++ b/examples/terraform/packet/variables.tf @@ -24,7 +24,6 @@ variable "worker_os" { # valid choices are: # * ubuntu # * centos - # * coreos default = "ubuntu" } diff --git a/examples/terraform/vsphere/variables.tf b/examples/terraform/vsphere/variables.tf index 6f3f0724c..696a63314 100644 --- a/examples/terraform/vsphere/variables.tf +++ b/examples/terraform/vsphere/variables.tf @@ -24,7 +24,6 @@ variable "worker_os" { # valid choices are: # * ubuntu # * centos - # * coreos default = "ubuntu" } diff --git a/hack/images/kubeone-e2e/Dockerfile b/hack/images/kubeone-e2e/Dockerfile index b24100021..d025d511d 100644 --- a/hack/images/kubeone-e2e/Dockerfile +++ b/hack/images/kubeone-e2e/Dockerfile @@ -14,13 +14,13 @@ # building image -FROM golang:1.15.2 as builder +FROM golang:1.15.7 as builder RUN apt-get update && apt-get install -y \ unzip \ upx-ucl -ENV TERRAFORM_VERSION "0.12.29" +ENV TERRAFORM_VERSION "0.12.30" RUN curl -fL https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip | funzip >/usr/local/bin/terraform RUN chmod +x /usr/local/bin/terraform @@ -37,11 +37,11 @@ RUN /opt/install-kube-tests-binaries.sh # resulting image -FROM golang:1.15.2 +FROM golang:1.15.7 ARG version -LABEL "io.kubeone"="Loodse GmbH" +LABEL "io.kubeone"="Kubermatic GmbH" LABEL version=${version} LABEL description="Set of kubernetes binaries to conduct kubeone E2E tests" LABEL maintainer="https://github.com/kubermatic/kubeone/blob/master/OWNERS" diff --git a/hack/images/kubeone-e2e/install-kube-tests-binaries.sh b/hack/images/kubeone-e2e/install-kube-tests-binaries.sh index 7599ad943..d7ea8f86d 100755 --- a/hack/images/kubeone-e2e/install-kube-tests-binaries.sh +++ b/hack/images/kubeone-e2e/install-kube-tests-binaries.sh @@ -17,10 +17,9 @@ set -euox pipefail declare -A full_versions -full_versions["1.16"]="v1.16.15" -full_versions["1.17"]="v1.17.12" -full_versions["1.18"]="v1.18.9" -full_versions["1.19"]="v1.19.2" +full_versions["1.18"]="v1.18.15" +full_versions["1.19"]="v1.19.7" +full_versions["1.20"]="v1.20.2" root_dir=${KUBETESTS_ROOT:-"/opt/kube-test"} tmp_root=${TMP_ROOT:-"/tmp/get-kube"} diff --git a/hack/images/kubeone-e2e/release.sh b/hack/images/kubeone-e2e/release.sh index 2929cab06..3e2008978 100755 --- a/hack/images/kubeone-e2e/release.sh +++ b/hack/images/kubeone-e2e/release.sh @@ -16,7 +16,7 @@ set -euox pipefail -TAG=v0.1.12 +TAG=v0.1.13 docker build --build-arg version=${TAG} --pull -t kubermatic/kubeone-e2e:${TAG} . docker push kubermatic/kubeone-e2e:${TAG} diff --git a/install.sh b/install.sh index 9b93a3bbb..15374eb9e 100755 --- a/install.sh +++ b/install.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/usr/bin/env sh # Copyright 2019 The KubeOne Authors. # @@ -14,38 +14,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -# -# This is a simple installer script for KubeOne # -# +# This is a simple installer script for KubeOne. + +set -eu # What OS is used -OS=$(uname) -# find out what's the latest version -VERSION=$(curl -w '%{url_effective}' -I -L -s -S https://github.com/kubermatic/kubeone/releases/latest -o /dev/null | sed -e 's|.*/v||') -# download URL for the latest version +OS="$(uname | tr '[:upper:]' '[:lower:]')" +# Find out what's the latest version +VERSION="$(curl -w '%{url_effective}' -I -L -s -S https://github.com/kubermatic/kubeone/releases/latest -o /dev/null | sed -e 's|.*/v||')" +# Download URL for the latest version URL="https://github.com/kubermatic/kubeone/releases/download/v${VERSION}/kubeone_${VERSION}_${OS}_amd64.zip" -# 'kubeone' will be installed into this dir: +# 'kubeone' will be installed into this dir DEST=/usr/local/bin # Download the latest version for the OS and save it as zip - if curl -LO "$URL" -then - echo "Copying kubeone binary into $DEST" - # unpack: - - - if unzip "kubeone_${VERSION}_${OS}_amd64.zip" -d "kubeone_${VERSION}_${OS}_amd64" - then - sudo mv "kubeone_${VERSION}_${OS}_amd64/kubeone" "$DEST" - rm "kubeone_${VERSION}_${OS}_amd64.zip" - rm -rf "kubeone_${VERSION}_${OS}_amd64" - echo "kubeone has been installed into $DEST/kubeone" - exit 0 - fi +then + echo "Copying kubeone binary into $DEST" + + if unzip "kubeone_${VERSION}_${OS}_amd64.zip" -d "kubeone_${VERSION}_${OS}_amd64" + then + sudo mv "kubeone_${VERSION}_${OS}_amd64/kubeone" "$DEST" + rm "kubeone_${VERSION}_${OS}_amd64.zip" + echo "Kubermatic KubeOne has been installed into $DEST/kubeone" + echo "Terraform example configs, addons, and helper scripts have been downloaded into the ./kubeone_${VERSION}_${OS}_amd64 directory" + exit 0 + fi else - printf "Failed to determine your platform.\n Try downloading from https://github.com/kubermatic/kubeone/releases" + printf "Failed to determine your platform.\n Try downloading from https://github.com/kubermatic/kubeone/releases" fi exit 1 diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index acf3a882b..6c0c2356e 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -49,10 +49,11 @@ func Ensure(s *state.State) error { } s.Logger.Infoln("Applying addons...") - creds, err := credentials.ProviderCredentials(s.Cluster.CloudProvider, s.CredentialsFilePath) + creds, err := credentials.Any(s.CredentialsFilePath) if err != nil { return errors.Wrap(err, "unable to fetch credentials") } + templateData := TemplateData{ Config: s.Cluster, Credentials: creds, diff --git a/pkg/apis/kubeone/types.go b/pkg/apis/kubeone/types.go index b1d558ef7..a5e4fb0db 100644 --- a/pkg/apis/kubeone/types.go +++ b/pkg/apis/kubeone/types.go @@ -78,12 +78,12 @@ type ContainerRuntimeContainerd struct{} // OperatingSystemName defines the operating system used on instances type OperatingSystemName string -var ( +const ( OperatingSystemNameUbuntu OperatingSystemName = "ubuntu" + OperatingSystemNameDebian OperatingSystemName = "debian" OperatingSystemNameCentOS OperatingSystemName = "centos" OperatingSystemNameRHEL OperatingSystemName = "rhel" OperatingSystemNameAmazon OperatingSystemName = "amzn" - OperatingSystemNameCoreOS OperatingSystemName = "coreos" OperatingSystemNameFlatcar OperatingSystemName = "flatcar" OperatingSystemNameUnknown OperatingSystemName = "" ) @@ -351,6 +351,7 @@ type Features struct { // PodNodeSelector PodNodeSelector *PodNodeSelector `json:"podNodeSelector,omitempty"` // PodPresets + // Deprecated: will be removed once Kubernetes 1.19 reaches EOL PodPresets *PodPresets `json:"podPresets,omitempty"` // PodSecurityPolicy PodSecurityPolicy *PodSecurityPolicy `json:"podSecurityPolicy,omitempty"` @@ -473,6 +474,9 @@ type PodNodeSelectorConfig struct { } // PodPresets feature flag +// The PodPresets feature has been removed in Kubernetes 1.20. +// This feature is deprecated and will be removed from the API once +// Kubernetes 1.19 reaches EOL. type PodPresets struct { // Enable Enable bool `json:"enable,omitempty"` diff --git a/pkg/apis/kubeone/v1alpha1/defaults.go b/pkg/apis/kubeone/v1alpha1/defaults.go index 1cd78a941..ed98e37fc 100644 --- a/pkg/apis/kubeone/v1alpha1/defaults.go +++ b/pkg/apis/kubeone/v1alpha1/defaults.go @@ -90,9 +90,7 @@ func SetDefaults_APIEndpoints(obj *KubeOneCluster) { } obj.APIEndpoint.Host = obj.Hosts[0].PublicAddress } - if obj.APIEndpoint.Port == 0 { - obj.APIEndpoint.Port = 6443 - } + obj.APIEndpoint.Port = defaulti(obj.APIEndpoint.Port, 6443) } func SetDefaults_Versions(obj *KubeOneCluster) { @@ -101,18 +99,10 @@ func SetDefaults_Versions(obj *KubeOneCluster) { } func SetDefaults_ClusterNetwork(obj *KubeOneCluster) { - if len(obj.ClusterNetwork.PodSubnet) == 0 { - obj.ClusterNetwork.PodSubnet = DefaultPodSubnet - } - if len(obj.ClusterNetwork.ServiceSubnet) == 0 { - obj.ClusterNetwork.ServiceSubnet = DefaultServiceSubnet - } - if len(obj.ClusterNetwork.ServiceDomainName) == 0 { - obj.ClusterNetwork.ServiceDomainName = DefaultServiceDNS - } - if len(obj.ClusterNetwork.NodePortRange) == 0 { - obj.ClusterNetwork.NodePortRange = DefaultNodePortRange - } + obj.ClusterNetwork.PodSubnet = defaults(obj.ClusterNetwork.PodSubnet, DefaultPodSubnet) + obj.ClusterNetwork.ServiceSubnet = defaults(obj.ClusterNetwork.ServiceSubnet, DefaultServiceSubnet) + obj.ClusterNetwork.ServiceDomainName = defaults(obj.ClusterNetwork.ServiceDomainName, DefaultServiceDNS) + obj.ClusterNetwork.NodePortRange = defaults(obj.ClusterNetwork.NodePortRange, DefaultNodePortRange) if obj.ClusterNetwork.CNI == nil { obj.ClusterNetwork.CNI = &CNI{ Provider: CNIProviderCanal, @@ -165,29 +155,31 @@ func SetDefaults_Features(obj *KubeOneCluster) { if obj.Features.StaticAuditLog != nil && obj.Features.StaticAuditLog.Enable { defaultStaticAuditLogConfig(&obj.Features.StaticAuditLog.Config) } + if obj.Features.OpenIDConnect != nil && obj.Features.OpenIDConnect.Enable { + defaultOpenIDConnect(&obj.Features.OpenIDConnect.Config) + } +} + +func defaultOpenIDConnect(config *OpenIDConnectConfig) { + config.ClientID = defaults(config.ClientID, "kubernetes") + config.UsernameClaim = defaults(config.UsernameClaim, "sub") + config.UsernamePrefix = defaults(config.UsernamePrefix, "oidc:") + config.GroupsClaim = defaults(config.GroupsClaim, "groups") + config.GroupsPrefix = defaults(config.GroupsPrefix, "oidc:") + config.SigningAlgs = defaults(config.SigningAlgs, "RS256") } func SetDefaults_Addons(obj *KubeOneCluster) { if obj.Addons != nil && obj.Addons.Enable { - if len(obj.Addons.Path) == 0 { - obj.Addons.Path = "./addons" - } + obj.Addons.Path = defaults(obj.Addons.Path, "./addons") } } func defaultStaticAuditLogConfig(obj *StaticAuditLogConfig) { - if obj.LogPath == "" { - obj.LogPath = "/var/log/kubernetes/audit.log" - } - if obj.LogMaxAge == 0 { - obj.LogMaxAge = 30 - } - if obj.LogMaxBackup == 0 { - obj.LogMaxBackup = 3 - } - if obj.LogMaxSize == 0 { - obj.LogMaxSize = 100 - } + obj.LogPath = defaults(obj.LogPath, "/var/log/kubernetes/audit.log") + obj.LogMaxAge = defaulti(obj.LogMaxAge, 30) + obj.LogMaxBackup = defaulti(obj.LogMaxBackup, 3) + obj.LogMaxSize = defaulti(obj.LogMaxSize, 100) } func defaultHostConfig(obj *HostConfig) { @@ -197,19 +189,25 @@ func defaultHostConfig(obj *HostConfig) { if len(obj.PrivateAddress) == 0 && len(obj.PublicAddress) > 0 { obj.PrivateAddress = obj.PublicAddress } - if len(obj.SSHPrivateKeyFile) == 0 && len(obj.SSHAgentSocket) == 0 { - obj.SSHAgentSocket = "env:SSH_AUTH_SOCK" - } - if obj.SSHUsername == "" { - obj.SSHUsername = "root" - } - if obj.SSHPort == 0 { - obj.SSHPort = 22 + if obj.SSHPrivateKeyFile == "" { + obj.SSHAgentSocket = defaults(obj.SSHAgentSocket, "env:SSH_AUTH_SOCK") } - if obj.BastionPort == 0 { - obj.BastionPort = 22 + obj.SSHUsername = defaults(obj.SSHUsername, "root") + obj.SSHPort = defaulti(obj.SSHPort, 22) + obj.BastionPort = defaulti(obj.BastionPort, 22) + obj.BastionUser = defaults(obj.BastionUser, obj.SSHUsername) +} + +func defaults(input string, defaultValue string) string { + if input != "" { + return input } - if obj.BastionUser == "" { - obj.BastionUser = obj.SSHUsername + return defaultValue +} + +func defaulti(input, defaultValue int) int { + if input != 0 { + return input } + return defaultValue } diff --git a/pkg/apis/kubeone/v1alpha1/types.go b/pkg/apis/kubeone/v1alpha1/types.go index 81cc621c7..334749e54 100644 --- a/pkg/apis/kubeone/v1alpha1/types.go +++ b/pkg/apis/kubeone/v1alpha1/types.go @@ -215,7 +215,8 @@ type MachineControllerConfig struct { // Features controls what features will be enabled on the cluster type Features struct { - PodNodeSelector *PodNodeSelector `json:"podNodeSelector"` + PodNodeSelector *PodNodeSelector `json:"podNodeSelector"` + // Deprecated: will be removed once Kubernetes 1.19 reaches EOL PodPresets *PodPresets `json:"podPresets"` PodSecurityPolicy *PodSecurityPolicy `json:"podSecurityPolicy"` StaticAuditLog *StaticAuditLog `json:"staticAuditLog"` @@ -247,6 +248,9 @@ type PodNodeSelectorConfig struct { } // PodPresets feature flag +// The PodPresets feature has been removed in Kubernetes 1.20. +// This feature is deprecated and will be removed from the API once +// Kubernetes 1.19 reaches EOL. type PodPresets struct { Enable bool `json:"enable"` } diff --git a/pkg/apis/kubeone/v1beta1/defaults.go b/pkg/apis/kubeone/v1beta1/defaults.go index 1a1c709ce..062d2e83e 100644 --- a/pkg/apis/kubeone/v1beta1/defaults.go +++ b/pkg/apis/kubeone/v1beta1/defaults.go @@ -109,9 +109,7 @@ func SetDefaults_APIEndpoints(obj *KubeOneCluster) { } obj.APIEndpoint.Host = obj.ControlPlane.Hosts[0].PublicAddress } - if obj.APIEndpoint.Port == 0 { - obj.APIEndpoint.Port = 6443 - } + obj.APIEndpoint.Port = defaulti(obj.APIEndpoint.Port, 6443) } func SetDefaults_Versions(obj *KubeOneCluster) { @@ -139,29 +137,21 @@ func SetDefaults_ContainerRuntime(obj *KubeOneCluster) { } func SetDefaults_ClusterNetwork(obj *KubeOneCluster) { - if len(obj.ClusterNetwork.PodSubnet) == 0 { - obj.ClusterNetwork.PodSubnet = DefaultPodSubnet - } - if len(obj.ClusterNetwork.ServiceSubnet) == 0 { - obj.ClusterNetwork.ServiceSubnet = DefaultServiceSubnet - } - if len(obj.ClusterNetwork.ServiceDomainName) == 0 { - obj.ClusterNetwork.ServiceDomainName = DefaultServiceDNS - } - if len(obj.ClusterNetwork.NodePortRange) == 0 { - obj.ClusterNetwork.NodePortRange = DefaultNodePortRange - } + obj.ClusterNetwork.PodSubnet = defaults(obj.ClusterNetwork.PodSubnet, DefaultPodSubnet) + obj.ClusterNetwork.ServiceSubnet = defaults(obj.ClusterNetwork.ServiceSubnet, DefaultServiceSubnet) + obj.ClusterNetwork.ServiceDomainName = defaults(obj.ClusterNetwork.ServiceDomainName, DefaultServiceDNS) + obj.ClusterNetwork.NodePortRange = defaults(obj.ClusterNetwork.NodePortRange, DefaultNodePortRange) defaultCanal := &CanalSpec{MTU: DefaultCanalMTU} switch { case obj.CloudProvider.AWS != nil: - defaultCanal.MTU = 8951 // 9001 AWS Jumbo Frame - 50 VXLAN bytes + defaultCanal.MTU = defaulti(defaultCanal.MTU, 8951) // 9001 AWS Jumbo Frame - 50 VXLAN bytes case obj.CloudProvider.GCE != nil: - defaultCanal.MTU = 1410 // GCE specific 1460 bytes - 50 VXLAN bytes + defaultCanal.MTU = defaulti(defaultCanal.MTU, 1410) // GCE specific 1460 bytes - 50 VXLAN bytes case obj.CloudProvider.Hetzner != nil: - defaultCanal.MTU = 1400 // Hetzner specific 1450 bytes - 50 VXLAN bytes + defaultCanal.MTU = defaulti(defaultCanal.MTU, 1400) // Hetzner specific 1450 bytes - 50 VXLAN bytes case obj.CloudProvider.Openstack != nil: - defaultCanal.MTU = 1400 // Openstack specific 1450 bytes - 50 VXLAN bytes + defaultCanal.MTU = defaulti(defaultCanal.MTU, 1400) // Openstack specific 1450 bytes - 50 VXLAN bytes } if obj.ClusterNetwork.CNI == nil { @@ -213,18 +203,22 @@ func SetDefaults_AssetConfiguration(obj *KubeOneCluster) { return } - if obj.AssetConfiguration.Kubernetes.ImageRepository == "" { - obj.AssetConfiguration.Kubernetes.ImageRepository = obj.RegistryConfiguration.OverwriteRegistry - } - if obj.AssetConfiguration.CoreDNS.ImageRepository == "" { - obj.AssetConfiguration.CoreDNS.ImageRepository = obj.RegistryConfiguration.OverwriteRegistry - } - if obj.AssetConfiguration.Etcd.ImageRepository == "" { - obj.AssetConfiguration.Etcd.ImageRepository = obj.RegistryConfiguration.OverwriteRegistry - } - if obj.AssetConfiguration.MetricsServer.ImageRepository == "" { - obj.AssetConfiguration.MetricsServer.ImageRepository = obj.RegistryConfiguration.OverwriteRegistry - } + obj.AssetConfiguration.Kubernetes.ImageRepository = defaults( + obj.AssetConfiguration.Kubernetes.ImageRepository, + obj.RegistryConfiguration.OverwriteRegistry, + ) + obj.AssetConfiguration.CoreDNS.ImageRepository = defaults( + obj.AssetConfiguration.CoreDNS.ImageRepository, + obj.RegistryConfiguration.OverwriteRegistry, + ) + obj.AssetConfiguration.Etcd.ImageRepository = defaults( + obj.AssetConfiguration.Etcd.ImageRepository, + obj.RegistryConfiguration.OverwriteRegistry, + ) + obj.AssetConfiguration.MetricsServer.ImageRepository = defaults( + obj.AssetConfiguration.MetricsServer.ImageRepository, + obj.RegistryConfiguration.OverwriteRegistry, + ) } func SetDefaults_Features(obj *KubeOneCluster) { @@ -236,29 +230,31 @@ func SetDefaults_Features(obj *KubeOneCluster) { if obj.Features.StaticAuditLog != nil && obj.Features.StaticAuditLog.Enable { defaultStaticAuditLogConfig(&obj.Features.StaticAuditLog.Config) } + if obj.Features.OpenIDConnect != nil && obj.Features.OpenIDConnect.Enable { + defaultOpenIDConnect(&obj.Features.OpenIDConnect.Config) + } +} + +func defaultOpenIDConnect(config *OpenIDConnectConfig) { + config.ClientID = defaults(config.ClientID, "kubernetes") + config.UsernameClaim = defaults(config.UsernameClaim, "sub") + config.UsernamePrefix = defaults(config.UsernamePrefix, "oidc:") + config.GroupsClaim = defaults(config.GroupsClaim, "groups") + config.GroupsPrefix = defaults(config.GroupsPrefix, "oidc:") + config.SigningAlgs = defaults(config.SigningAlgs, "RS256") } func SetDefaults_Addons(obj *KubeOneCluster) { if obj.Addons != nil && obj.Addons.Enable { - if len(obj.Addons.Path) == 0 { - obj.Addons.Path = "./addons" - } + obj.Addons.Path = defaults(obj.Addons.Path, "./addons") } } func defaultStaticAuditLogConfig(obj *StaticAuditLogConfig) { - if obj.LogPath == "" { - obj.LogPath = "/var/log/kubernetes/audit.log" - } - if obj.LogMaxAge == 0 { - obj.LogMaxAge = 30 - } - if obj.LogMaxBackup == 0 { - obj.LogMaxBackup = 3 - } - if obj.LogMaxSize == 0 { - obj.LogMaxSize = 100 - } + obj.LogPath = defaults(obj.LogPath, "/var/log/kubernetes/audit.log") + obj.LogMaxAge = defaulti(obj.LogMaxAge, 30) + obj.LogMaxBackup = defaulti(obj.LogMaxBackup, 3) + obj.LogMaxSize = defaulti(obj.LogMaxSize, 100) } func defaultHostConfig(obj *HostConfig) { @@ -268,19 +264,25 @@ func defaultHostConfig(obj *HostConfig) { if len(obj.PrivateAddress) == 0 && len(obj.PublicAddress) > 0 { obj.PrivateAddress = obj.PublicAddress } - if len(obj.SSHPrivateKeyFile) == 0 && len(obj.SSHAgentSocket) == 0 { - obj.SSHAgentSocket = "env:SSH_AUTH_SOCK" - } - if obj.SSHUsername == "" { - obj.SSHUsername = "root" + if obj.SSHPrivateKeyFile == "" { + obj.SSHAgentSocket = defaults(obj.SSHAgentSocket, "env:SSH_AUTH_SOCK") } - if obj.SSHPort == 0 { - obj.SSHPort = 22 - } - if obj.BastionPort == 0 { - obj.BastionPort = 22 + obj.SSHUsername = defaults(obj.SSHUsername, "root") + obj.SSHPort = defaulti(obj.SSHPort, 22) + obj.BastionPort = defaulti(obj.BastionPort, 22) + obj.BastionUser = defaults(obj.BastionUser, obj.SSHUsername) +} + +func defaults(input, defaultValue string) string { + if input != "" { + return input } - if obj.BastionUser == "" { - obj.BastionUser = obj.SSHUsername + return defaultValue +} + +func defaulti(input, defaultValue int) int { + if input != 0 { + return input } + return defaultValue } diff --git a/pkg/apis/kubeone/v1beta1/types.go b/pkg/apis/kubeone/v1beta1/types.go index 87efb9187..a975f323a 100644 --- a/pkg/apis/kubeone/v1beta1/types.go +++ b/pkg/apis/kubeone/v1beta1/types.go @@ -78,12 +78,12 @@ type ContainerRuntimeContainerd struct{} // OperatingSystemName defines the operating system used on instances type OperatingSystemName string -var ( +const ( OperatingSystemNameUbuntu OperatingSystemName = "ubuntu" + OperatingSystemNameDebian OperatingSystemName = "debian" OperatingSystemNameCentOS OperatingSystemName = "centos" OperatingSystemNameRHEL OperatingSystemName = "rhel" OperatingSystemNameAmazon OperatingSystemName = "amzn" - OperatingSystemNameCoreOS OperatingSystemName = "coreos" OperatingSystemNameFlatcar OperatingSystemName = "flatcar" OperatingSystemNameUnknown OperatingSystemName = "" ) @@ -351,6 +351,7 @@ type Features struct { // PodNodeSelector PodNodeSelector *PodNodeSelector `json:"podNodeSelector,omitempty"` // PodPresets + // Deprecated: will be removed once Kubernetes 1.19 reaches EOL PodPresets *PodPresets `json:"podPresets,omitempty"` // PodSecurityPolicy PodSecurityPolicy *PodSecurityPolicy `json:"podSecurityPolicy,omitempty"` @@ -473,6 +474,9 @@ type PodNodeSelectorConfig struct { } // PodPresets feature flag +// The PodPresets feature has been removed in Kubernetes 1.20. +// This feature is deprecated and will be removed from the API once +// Kubernetes 1.19 reaches EOL. type PodPresets struct { // Enable Enable bool `json:"enable,omitempty"` diff --git a/pkg/apis/kubeone/validation/validation.go b/pkg/apis/kubeone/validation/validation.go index eb52bc05f..92727592a 100644 --- a/pkg/apis/kubeone/validation/validation.go +++ b/pkg/apis/kubeone/validation/validation.go @@ -50,7 +50,7 @@ func ValidateKubeOneCluster(c kubeone.KubeOneCluster) field.ErrorList { "machine-controller deployment is disabled, but the configuration still contains dynamic workers")) } - allErrs = append(allErrs, ValidateFeatures(c.Features, field.NewPath("features"))...) + allErrs = append(allErrs, ValidateFeatures(c.Features, c.Versions, field.NewPath("features"))...) allErrs = append(allErrs, ValidateAddons(c.Addons, field.NewPath("addons"))...) allErrs = append(allErrs, ValidateRegistryConfiguration(c.RegistryConfiguration, field.NewPath("registryConfiguration"))...) @@ -294,7 +294,7 @@ func ValidateDynamicWorkerConfig(workerset []kubeone.DynamicWorkerConfig, fldPat } // ValidateFeatures validates the Features structure -func ValidateFeatures(f kubeone.Features, fldPath *field.Path) field.ErrorList { +func ValidateFeatures(f kubeone.Features, versions kubeone.VersionConfig, fldPath *field.Path) field.ErrorList { allErrs := field.ErrorList{} if f.PodNodeSelector != nil && f.PodNodeSelector.Enable { @@ -306,6 +306,13 @@ func ValidateFeatures(f kubeone.Features, fldPath *field.Path) field.ErrorList { if f.OpenIDConnect != nil && f.OpenIDConnect.Enable { allErrs = append(allErrs, ValidateOIDCConfig(f.OpenIDConnect.Config, fldPath.Child("openidConnect"))...) } + if f.PodPresets != nil && f.PodPresets.Enable { + kubeVer, _ := semver.NewVersion(versions.Kubernetes) + gteKube120Condition, _ := semver.NewConstraint(">= 1.20") + if gteKube120Condition.Check(kubeVer) { + allErrs = append(allErrs, field.Forbidden(fldPath.Child("podPresets"), "podPresets feature is removed in kubernetes 1.20+ and must be disabled")) + } + } return allErrs } diff --git a/pkg/apis/kubeone/validation/validation_test.go b/pkg/apis/kubeone/validation/validation_test.go index 46ba0ba5b..54bfc9d11 100644 --- a/pkg/apis/kubeone/validation/validation_test.go +++ b/pkg/apis/kubeone/validation/validation_test.go @@ -1003,6 +1003,7 @@ func TestValidateFeatures(t *testing.T) { tests := []struct { name string features kubeone.Features + versions kubeone.VersionConfig expectedError bool }{ { @@ -1015,6 +1016,9 @@ func TestValidateFeatures(t *testing.T) { Enable: true, }, }, + versions: kubeone.VersionConfig{ + Kubernetes: "1.20.2", + }, expectedError: false, }, { @@ -1024,11 +1028,17 @@ func TestValidateFeatures(t *testing.T) { Enable: false, }, }, + versions: kubeone.VersionConfig{ + Kubernetes: "1.20.2", + }, expectedError: false, }, { - name: "no feature configured", - features: kubeone.Features{}, + name: "no feature configured", + features: kubeone.Features{}, + versions: kubeone.VersionConfig{ + Kubernetes: "1.20.2", + }, expectedError: false, }, { @@ -1043,6 +1053,9 @@ func TestValidateFeatures(t *testing.T) { }, }, }, + versions: kubeone.VersionConfig{ + Kubernetes: "1.20.2", + }, expectedError: false, }, { @@ -1053,6 +1066,9 @@ func TestValidateFeatures(t *testing.T) { Config: kubeone.StaticAuditLogConfig{}, }, }, + versions: kubeone.VersionConfig{ + Kubernetes: "1.20.2", + }, expectedError: true, }, { @@ -1063,6 +1079,9 @@ func TestValidateFeatures(t *testing.T) { Config: kubeone.OpenIDConnectConfig{}, }, }, + versions: kubeone.VersionConfig{ + Kubernetes: "1.20.2", + }, expectedError: true, }, { @@ -1073,13 +1092,52 @@ func TestValidateFeatures(t *testing.T) { Config: kubeone.PodNodeSelectorConfig{}, }, }, + versions: kubeone.VersionConfig{ + Kubernetes: "1.20.2", + }, + expectedError: true, + }, + { + name: "podPresets enabled on 1.19 cluster", + features: kubeone.Features{ + PodPresets: &kubeone.PodPresets{ + Enable: true, + }, + }, + versions: kubeone.VersionConfig{ + Kubernetes: "1.19.7", + }, + expectedError: false, + }, + { + name: "podPresets enabled on 1.20 cluster", + features: kubeone.Features{ + PodPresets: &kubeone.PodPresets{ + Enable: true, + }, + }, + versions: kubeone.VersionConfig{ + Kubernetes: "1.20.2", + }, + expectedError: true, + }, + { + name: "podPresets enabled on 1.21 cluster", + features: kubeone.Features{ + PodPresets: &kubeone.PodPresets{ + Enable: true, + }, + }, + versions: kubeone.VersionConfig{ + Kubernetes: "1.21.0", + }, expectedError: true, }, } for _, tc := range tests { tc := tc t.Run(tc.name, func(t *testing.T) { - errs := ValidateFeatures(tc.features, nil) + errs := ValidateFeatures(tc.features, tc.versions, nil) if (len(errs) == 0) == tc.expectedError { t.Errorf("test case failed: expected %v, but got %v", tc.expectedError, (len(errs) != 0)) } diff --git a/pkg/cmd/config.go b/pkg/cmd/config.go index bf84e8109..91c407928 100644 --- a/pkg/cmd/config.go +++ b/pkg/cmd/config.go @@ -565,6 +565,27 @@ cloudProvider: # Path to file that will be uploaded and used as custom '--cloud-config' file. cloudConfig: "{{ .CloudProviderCloudCfg }}" +# Controls which container runtime will be installed on instances. +# By default: +# * Docker will be installed for Kubernetes clusters up to 1.20 +# * containerd will be installed for Kubernetes clusters 1.21+ +# Currently, it's not possible to migrate existing clusters from one to another +# container runtime, however, migration from Docker to containerd is planned +# for one of the upcoming KubeOne releases. +# Only one container runtime can be present at the time. +# +# Note: Kubernetes has announced deprecation of Docker (dockershim) support. +# It's expected that the Docker support will be removed in Kubernetes 1.22. +# It's highly advised to use containerd for all newly created clusters. +containerRuntime: + # Installs containerd container runtime. + # Default for 1.21+ Kubernetes clusters. + # containerd: {} + # Installs Docker container runtime. + # Default for Kubernetes clusters up to 1.20. + # This option will be removed once Kubernetes 1.21 reaches EOL. + # docker: {} + features: # Enable the PodNodeSelector admission plugin in API server. # More info: https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#podnodeselector @@ -582,6 +603,11 @@ features: podSecurityPolicy: enable: {{ .EnablePodSecurityPolicy }} # Enables PodPresets admission plugin in API server. + # The PodPresets feature has been removed in Kubernetes 1.20. + # This feature is deprecated and will be removed from the API once + # Kubernetes 1.19 reaches EOL. + # Provisioning a Kubernetes 1.20 cluster or upgrading an existing cluster to + # the Kubernetes 1.20 requires this feature to be disabled. podPresets: enable: {{ .EnablePodPresets }} # Enables and configures audit log backend. diff --git a/pkg/credentials/credentials.go b/pkg/credentials/credentials.go index 4354b67df..3e261a50c 100644 --- a/pkg/credentials/credentials.go +++ b/pkg/credentials/credentials.go @@ -22,7 +22,7 @@ import ( "os" "strings" - "github.com/aws/aws-sdk-go/aws/credentials" + awscredentials "github.com/aws/aws-sdk-go/aws/credentials" "github.com/pkg/errors" "gopkg.in/yaml.v2" @@ -68,6 +68,32 @@ const ( VSphereUsernameMC = "VSPHERE_USERNAME" ) +var ( + allKeys = []string{ + AWSAccessKeyID, + AWSSecretAccessKey, + AzureClientID, + AzureClientSecret, + AzureTenantID, + AzureSubscribtionID, + DigitalOceanTokenKey, + GoogleServiceAccountKey, + HetznerTokenKey, + OpenStackAuthURL, + OpenStackDomainName, + OpenStackPassword, + OpenStackRegionName, + OpenStackTenantID, + OpenStackTenantName, + OpenStackUserName, + PacketAPIKey, + PacketProjectID, + VSphereAddress, + VSpherePassword, + VSphereUsername, + } +) + // ProviderEnvironmentVariable is used to match environment variable used by KubeOne to environment variable used by // machine-controller. type ProviderEnvironmentVariable struct { @@ -75,29 +101,46 @@ type ProviderEnvironmentVariable struct { MachineControllerName string } +func Any(credentialsFilePath string) (map[string]string, error) { + credentialsFinder, err := newCredsFinder(credentialsFilePath) + if err != nil { + return nil, err + } + + creds := map[string]string{} + + for _, key := range allKeys { + if val := credentialsFinder(key); val != "" { + creds[key] = val + } + } + + return creds, nil +} + // ProviderCredentials implements fetching credentials for each supported provider -func ProviderCredentials(p kubeone.CloudProviderSpec, credentialsFilePath string) (map[string]string, error) { - f, err := newFetcher(credentialsFilePath) +func ProviderCredentials(cloudProvider kubeone.CloudProviderSpec, credentialsFilePath string) (map[string]string, error) { + credentialsFinder, err := newCredsFinder(credentialsFilePath) if err != nil { return nil, err } switch { - case p.AWS != nil: - return f.parseAWSCredentials() - case p.Azure != nil: - return f.parseCredentialVariables([]ProviderEnvironmentVariable{ + case cloudProvider.AWS != nil: + return credentialsFinder.aws() + case cloudProvider.Azure != nil: + return credentialsFinder.parseCredentialVariables([]ProviderEnvironmentVariable{ {Name: AzureClientID, MachineControllerName: AzureClientIDMC}, {Name: AzureClientSecret, MachineControllerName: AzureClientSecretMC}, {Name: AzureTenantID, MachineControllerName: AzureTenantIDMC}, {Name: AzureSubscribtionID, MachineControllerName: AzureSubscribtionIDMC}, }, defaultValidationFunc) - case p.DigitalOcean != nil: - return f.parseCredentialVariables([]ProviderEnvironmentVariable{ + case cloudProvider.DigitalOcean != nil: + return credentialsFinder.parseCredentialVariables([]ProviderEnvironmentVariable{ {Name: DigitalOceanTokenKey, MachineControllerName: DigitalOceanTokenKeyMC}, }, defaultValidationFunc) - case p.GCE != nil: - gsa, err := f.parseCredentialVariables([]ProviderEnvironmentVariable{ + case cloudProvider.GCE != nil: + gsa, err := credentialsFinder.parseCredentialVariables([]ProviderEnvironmentVariable{ {Name: GoogleServiceAccountKey, MachineControllerName: GoogleServiceAccountKeyMC}, }, defaultValidationFunc) if err != nil { @@ -107,12 +150,12 @@ func ProviderCredentials(p kubeone.CloudProviderSpec, credentialsFilePath string // machine-controller, as machine-controller assumes it will be double encoded gsa[GoogleServiceAccountKeyMC] = base64.StdEncoding.EncodeToString([]byte(gsa[GoogleServiceAccountKeyMC])) return gsa, nil - case p.Hetzner != nil: - return f.parseCredentialVariables([]ProviderEnvironmentVariable{ + case cloudProvider.Hetzner != nil: + return credentialsFinder.parseCredentialVariables([]ProviderEnvironmentVariable{ {Name: HetznerTokenKey, MachineControllerName: HetznerTokenKeyMC}, }, defaultValidationFunc) - case p.Openstack != nil: - return f.parseCredentialVariables([]ProviderEnvironmentVariable{ + case cloudProvider.Openstack != nil: + return credentialsFinder.parseCredentialVariables([]ProviderEnvironmentVariable{ {Name: OpenStackAuthURL}, {Name: OpenStackUserName, MachineControllerName: OpenStackUserNameMC}, {Name: OpenStackPassword}, @@ -121,13 +164,13 @@ func ProviderCredentials(p kubeone.CloudProviderSpec, credentialsFilePath string {Name: OpenStackTenantID}, {Name: OpenStackTenantName}, }, openstackValidationFunc) - case p.Packet != nil: - return f.parseCredentialVariables([]ProviderEnvironmentVariable{ + case cloudProvider.Packet != nil: + return credentialsFinder.parseCredentialVariables([]ProviderEnvironmentVariable{ {Name: PacketAPIKey, MachineControllerName: PacketAPIKeyMC}, {Name: PacketProjectID}, }, defaultValidationFunc) - case p.Vsphere != nil: - vscreds, err := f.parseCredentialVariables([]ProviderEnvironmentVariable{ + case cloudProvider.Vsphere != nil: + vscreds, err := credentialsFinder.parseCredentialVariables([]ProviderEnvironmentVariable{ {Name: VSphereAddress, MachineControllerName: VSphereAddressMC}, {Name: VSphereUsername, MachineControllerName: VSphereUsernameMC}, {Name: VSpherePassword}, @@ -138,67 +181,50 @@ func ProviderCredentials(p kubeone.CloudProviderSpec, credentialsFilePath string // force scheme, as machine-controller requires it while terraform does not vscreds[VSphereAddressMC] = "https://" + vscreds[VSphereAddressMC] return vscreds, nil - case p.None != nil: + case cloudProvider.None != nil: return map[string]string{}, nil } return nil, errors.New("no provider matched") } -type fetcher struct { - // Source is custom source for credentials, by default environment is used - Source map[string]string - // F is function that retrieves variable from the source - F func(string) string -} +func newCredsFinder(credentialsFilePath string) (lookupFunc, error) { + staticMap := map[string]string{} + finder := func(name string) string { + if val := os.Getenv(name); val != "" { + return val + } + return staticMap[name] + } -func newFetcher(credentialsFilePath string) (*fetcher, error) { - f := &fetcher{ - F: os.Getenv, + if credentialsFilePath == "" { + return finder, nil } - if credentialsFilePath != "" { - b, err := ioutil.ReadFile(credentialsFilePath) - if err != nil { - return nil, errors.Wrap(err, "unable to load credentials file") - } - m := make(map[string]string) - err = yaml.Unmarshal(b, &m) - if err != nil { - return nil, errors.Wrap(err, "unable to unmarshal credentials file") - } - f.Source = m - f.F = func(name string) string { - return m[name] - } + buf, err := ioutil.ReadFile(credentialsFilePath) + if err != nil { + return nil, errors.Wrap(err, "unable to load credentials file") } - return f, nil + if err = yaml.Unmarshal(buf, &staticMap); err != nil { + return nil, errors.Wrap(err, "unable to unmarshal credentials file") + } + + return finder, nil } -func (f *fetcher) parseAWSCredentials() (map[string]string, error) { - if f.Source != nil { - return map[string]string{ - AWSAccessKeyID: f.F(AWSAccessKeyID), - AWSSecretAccessKey: f.F(AWSSecretAccessKey), - }, nil - } +// lookupFunc is function that retrieves credentials from the sources +type lookupFunc func(string) string +func (lookup lookupFunc) aws() (map[string]string, error) { creds := make(map[string]string) - envCredsProvider := credentials.NewEnvCredentials() + accessKeyID := lookup(AWSAccessKeyID) + secretAccessKey := lookup(AWSSecretAccessKey) - // will error out in case when ether ID or KEY are missing from ENV - envCreds, err := envCredsProvider.Get() - - switch err { - case nil: - creds[AWSAccessKeyID] = envCreds.AccessKeyID - creds[AWSSecretAccessKey] = envCreds.SecretAccessKey + if accessKeyID != "" && secretAccessKey != "" { + creds[AWSAccessKeyID] = accessKeyID + creds[AWSSecretAccessKey] = secretAccessKey return creds, nil - case credentials.ErrSecretAccessKeyNotFound, credentials.ErrAccessKeyIDNotFound: - // ignore above errors to continue to shared credentials method - default: - return nil, errors.WithStack(err) } if os.Getenv("AWS_PROFILE") == "" { @@ -209,10 +235,10 @@ func (f *fetcher) parseAWSCredentials() (map[string]string, error) { } // If env fails resort to config file - configCredsProvider := credentials.NewSharedCredentials("", "") + sharedCredsProvider := awscredentials.NewSharedCredentials("", "") // will error out in case when ether ID or KEY are missing from shared file - configCreds, err := configCredsProvider.Get() + configCreds, err := sharedCredsProvider.Get() if err != nil { return nil, errors.WithStack(err) } @@ -224,12 +250,13 @@ func (f *fetcher) parseAWSCredentials() (map[string]string, error) { return creds, nil } -func (f fetcher) parseCredentialVariables(envVars []ProviderEnvironmentVariable, validationFunc func(map[string]string) error) (map[string]string, error) { - // Validate credentials using given validation function +func (lookup lookupFunc) parseCredentialVariables(envVars []ProviderEnvironmentVariable, validationFunc func(map[string]string) error) (map[string]string, error) { creds := make(map[string]string) for _, env := range envVars { - creds[env.Name] = strings.TrimSpace(f.F(env.Name)) + creds[env.Name] = strings.TrimSpace(lookup(env.Name)) } + + // Validate credentials using given validation function if err := validationFunc(creds); err != nil { return nil, errors.Wrap(err, "unable to validate credentials") } diff --git a/pkg/scripts/node.go b/pkg/scripts/node.go index 195a902ec..38f172997 100644 --- a/pkg/scripts/node.go +++ b/pkg/scripts/node.go @@ -29,6 +29,28 @@ var ( sudo KUBECONFIG=/etc/kubernetes/admin.conf \ kubectl drain {{ .NODE_NAME }} --ignore-daemonsets --delete-local-data `) + + restartKubeAPIServerCrictlTemplate = heredoc.Doc(` + apiserver_id=$(sudo crictl ps --name=kube-apiserver -q) + [ -z "$apiserver_id" ] && exit 1 + + sudo crictl logs "$apiserver_id" > /tmp/kube-apiserver.log 2>&1 + if sudo grep -q "etcdserver: no leader" /tmp/kube-apiserver.log; then + sudo crictl rm "$apiserver_id" + sleep 10 + fi + `) + + restartKubeAPIServerDockerTemplate = heredoc.Doc(` + apiserver_id=$(sudo docker ps --filter="name=k8s_kube-apiserver" -q) + [ -z "$apiserver_id" ] && exit 1 + + sudo docker logs "$apiserver_id" > /tmp/kube-apiserver.log 2>&1 + if sudo grep -q "etcdserver: no leader" /tmp/kube-apiserver.log; then + sudo docker rm -f "$apiserver_id" + sleep 10 + fi + `) ) func DrainNode(nodeName string) (string, error) { @@ -40,3 +62,11 @@ func DrainNode(nodeName string) (string, error) { func Hostname() string { return hostnameScript } + +func RestartKubeAPIServerCrictl() string { + return restartKubeAPIServerCrictlTemplate +} + +func RestartKubeAPIServerDocker() string { + return restartKubeAPIServerDockerTemplate +} diff --git a/pkg/scripts/os.go b/pkg/scripts/os.go index c513d2e8f..e315412c7 100644 --- a/pkg/scripts/os.go +++ b/pkg/scripts/os.go @@ -147,7 +147,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync {{ if .INSTALL_DOCKER }} {{ template "docker-daemon-config" . }} @@ -222,7 +223,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync {{ if .INSTALL_DOCKER }} {{ template "docker-daemon-config" . }} @@ -307,7 +309,7 @@ sudo systemctl restart kubelet {{- end }} ` - kubeadmCoreOSTemplate = ` + kubeadmFlatcarTemplate = ` source /etc/kubeone/proxy-env {{ template "detect-host-cpu-architecture" }} @@ -400,7 +402,7 @@ sudo rm -f /etc/systemd/system/kubelet.service /etc/systemd/system/kubelet.servi sudo systemctl daemon-reload ` - removeBinariesCoreOSScriptTemplate = ` + removeBinariesFlatcarScriptTemplate = ` # Stop kubelet sudo systemctl stop kubelet || true # Remove CNI and binaries @@ -411,7 +413,7 @@ sudo rm -f /etc/systemd/system/kubelet.service /etc/systemd/system/kubelet.servi sudo systemctl daemon-reload ` - upgradeKubeadmAndCNICoreOSScriptTemplate = ` + upgradeKubeadmAndCNIFlatcarScriptTemplate = ` {{ template "detect-host-cpu-architecture" }} source /etc/kubeone/proxy-env @@ -433,7 +435,7 @@ sudo mv /var/tmp/kube-binaries/kubeadm . sudo chmod +x kubeadm ` - upgradeKubeletAndKubectlCoreOSScriptTemplate = ` + upgradeKubeletAndKubectlFlatcarScriptTemplate = ` source /etc/kubeone/proxy-env {{ template "detect-host-cpu-architecture" }} @@ -547,8 +549,8 @@ func KubeadmAmazonLinux(cluster *kubeone.KubeOneCluster, force bool) (string, er }) } -func KubeadmCoreOS(cluster *kubeone.KubeOneCluster) (string, error) { - return Render(kubeadmCoreOSTemplate, Data{ +func KubeadmFlatcar(cluster *kubeone.KubeOneCluster) (string, error) { + return Render(kubeadmFlatcarTemplate, Data{ "KUBERNETES_VERSION": cluster.Versions.Kubernetes, "KUBERNETES_CNI_VERSION": defaultKubernetesCNIVersion, "INSECURE_REGISTRY": cluster.RegistryConfiguration.InsecureRegistryAddress(), @@ -567,8 +569,8 @@ func RemoveBinariesAmazonLinux() (string, error) { return Render(removeBinariesAmazonLinuxScriptTemplate, Data{}) } -func RemoveBinariesCoreOS() (string, error) { - return Render(removeBinariesCoreOSScriptTemplate, nil) +func RemoveBinariesFlatcar() (string, error) { + return Render(removeBinariesFlatcarScriptTemplate, nil) } func UpgradeKubeadmAndCNIDebian(cluster *kubeone.KubeOneCluster) (string, error) { @@ -626,8 +628,8 @@ func UpgradeKubeadmAndCNIAmazonLinux(cluster *kubeone.KubeOneCluster) (string, e }) } -func UpgradeKubeadmAndCNICoreOS(k8sVersion string) (string, error) { - return Render(upgradeKubeadmAndCNICoreOSScriptTemplate, Data{ +func UpgradeKubeadmAndCNIFlatcar(k8sVersion string) (string, error) { + return Render(upgradeKubeadmAndCNIFlatcarScriptTemplate, Data{ "KUBERNETES_VERSION": k8sVersion, "KUBERNETES_CNI_VERSION": defaultKubernetesCNIVersion, }) @@ -691,8 +693,8 @@ func UpgradeKubeletAndKubectlAmazonLinux(cluster *kubeone.KubeOneCluster) (strin }) } -func UpgradeKubeletAndKubectlCoreOS(k8sVersion string) (string, error) { - return Render(upgradeKubeletAndKubectlCoreOSScriptTemplate, Data{ +func UpgradeKubeletAndKubectlFlatcar(k8sVersion string) (string, error) { + return Render(upgradeKubeletAndKubectlFlatcarScriptTemplate, Data{ "KUBERNETES_VERSION": k8sVersion, }) } diff --git a/pkg/scripts/os_test.go b/pkg/scripts/os_test.go index 007e1e910..9dc17028a 100644 --- a/pkg/scripts/os_test.go +++ b/pkg/scripts/os_test.go @@ -337,7 +337,7 @@ func TestKubeadmAmazonLinux(t *testing.T) { } } -func TestKubeadmCoreOS(t *testing.T) { +func TestKubeadmFlatcar(t *testing.T) { t.Parallel() type args struct { @@ -377,9 +377,9 @@ func TestKubeadmCoreOS(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - got, err := KubeadmCoreOS(&tt.args.cluster) + got, err := KubeadmFlatcar(&tt.args.cluster) if err != tt.err { - t.Errorf("KubeadmCoreOS() error = %v, wantErr %v", err, tt.err) + t.Errorf("KubeadmFlatcar() error = %v, wantErr %v", err, tt.err) return } @@ -424,12 +424,12 @@ func TestRemoveBinariesAmazonLinux(t *testing.T) { testhelper.DiffOutput(t, testhelper.FSGoldenName(t), got, *updateFlag) } -func TestRemoveBinariesCoreOS(t *testing.T) { +func TestRemoveBinariesFlatcar(t *testing.T) { t.Parallel() - got, err := RemoveBinariesCoreOS() + got, err := RemoveBinariesFlatcar() if err != nil { - t.Errorf("RemoveBinariesCoreOS() error = %v", err) + t.Errorf("RemoveBinariesFlatcar() error = %v", err) return } @@ -475,12 +475,12 @@ func TestUpgradeKubeadmAndCNIAmazonLinux(t *testing.T) { testhelper.DiffOutput(t, testhelper.FSGoldenName(t), got, *updateFlag) } -func TestUpgradeKubeadmAndCNICoreOS(t *testing.T) { +func TestUpgradeKubeadmAndCNIFlatcar(t *testing.T) { t.Parallel() - got, err := UpgradeKubeadmAndCNICoreOS("v1.17.4") + got, err := UpgradeKubeadmAndCNIFlatcar("v1.17.4") if err != nil { - t.Errorf("UpgradeKubeadmAndCNICoreOS() error = %v", err) + t.Errorf("UpgradeKubeadmAndCNIFlatcar() error = %v", err) return } @@ -526,12 +526,12 @@ func TestUpgradeKubeletAndKubectlAmazonLinux(t *testing.T) { testhelper.DiffOutput(t, testhelper.FSGoldenName(t), got, *updateFlag) } -func TestUpgradeKubeletAndKubectlCoreOS(t *testing.T) { +func TestUpgradeKubeletAndKubectlFlatcar(t *testing.T) { t.Parallel() - got, err := UpgradeKubeletAndKubectlCoreOS("v1.17.4") + got, err := UpgradeKubeletAndKubectlFlatcar("v1.17.4") if err != nil { - t.Errorf("UpgradeKubeletAndKubectlCoreOS() error = %v", err) + t.Errorf("UpgradeKubeletAndKubectlFlatcar() error = %v", err) return } diff --git a/pkg/scripts/testdata/TestKubeadmAmazonLinux-force.golden b/pkg/scripts/testdata/TestKubeadmAmazonLinux-force.golden index 40162fb5e..3a57dc7cc 100644 --- a/pkg/scripts/testdata/TestKubeadmAmazonLinux-force.golden +++ b/pkg/scripts/testdata/TestKubeadmAmazonLinux-force.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmAmazonLinux-overwrite_registry.golden b/pkg/scripts/testdata/TestKubeadmAmazonLinux-overwrite_registry.golden index b3caf8861..b3f705990 100644 --- a/pkg/scripts/testdata/TestKubeadmAmazonLinux-overwrite_registry.golden +++ b/pkg/scripts/testdata/TestKubeadmAmazonLinux-overwrite_registry.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmAmazonLinux-overwrite_registry_insecure.golden b/pkg/scripts/testdata/TestKubeadmAmazonLinux-overwrite_registry_insecure.golden index 1b55d985d..edd5faec0 100644 --- a/pkg/scripts/testdata/TestKubeadmAmazonLinux-overwrite_registry_insecure.golden +++ b/pkg/scripts/testdata/TestKubeadmAmazonLinux-overwrite_registry_insecure.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmAmazonLinux-proxy.golden b/pkg/scripts/testdata/TestKubeadmAmazonLinux-proxy.golden index 30a09704f..27bd65792 100644 --- a/pkg/scripts/testdata/TestKubeadmAmazonLinux-proxy.golden +++ b/pkg/scripts/testdata/TestKubeadmAmazonLinux-proxy.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmAmazonLinux-simple.golden b/pkg/scripts/testdata/TestKubeadmAmazonLinux-simple.golden index b3caf8861..b3f705990 100644 --- a/pkg/scripts/testdata/TestKubeadmAmazonLinux-simple.golden +++ b/pkg/scripts/testdata/TestKubeadmAmazonLinux-simple.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmAmazonLinux-v1.16.1.golden b/pkg/scripts/testdata/TestKubeadmAmazonLinux-v1.16.1.golden index 1667f44c2..8e3f68ee5 100644 --- a/pkg/scripts/testdata/TestKubeadmAmazonLinux-v1.16.1.golden +++ b/pkg/scripts/testdata/TestKubeadmAmazonLinux-v1.16.1.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmAmazonLinux-with_containerd.golden b/pkg/scripts/testdata/TestKubeadmAmazonLinux-with_containerd.golden index 650e7501d..63a28000a 100644 --- a/pkg/scripts/testdata/TestKubeadmAmazonLinux-with_containerd.golden +++ b/pkg/scripts/testdata/TestKubeadmAmazonLinux-with_containerd.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmAmazonLinux-with_containerd_with_insecure_registry.golden b/pkg/scripts/testdata/TestKubeadmAmazonLinux-with_containerd_with_insecure_registry.golden index d384a7823..61809b95d 100644 --- a/pkg/scripts/testdata/TestKubeadmAmazonLinux-with_containerd_with_insecure_registry.golden +++ b/pkg/scripts/testdata/TestKubeadmAmazonLinux-with_containerd_with_insecure_registry.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-force.golden b/pkg/scripts/testdata/TestKubeadmCentOS-force.golden index d089bd893..ab932341e 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-force.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-force.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry.golden b/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry.golden index 8402d13d2..2773395c2 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry_insecure.golden b/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry_insecure.golden index a2388a430..44f743ff0 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry_insecure.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-overwrite_registry_insecure.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-proxy.golden b/pkg/scripts/testdata/TestKubeadmCentOS-proxy.golden index 9998c09e6..67cdd15de 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-proxy.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-proxy.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-simple.golden b/pkg/scripts/testdata/TestKubeadmCentOS-simple.golden index 8402d13d2..2773395c2 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-simple.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-simple.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-v1.16.1.golden b/pkg/scripts/testdata/TestKubeadmCentOS-v1.16.1.golden index 02cbf4f9c..de24698a6 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-v1.16.1.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-v1.16.1.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd.golden b/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd.golden index e10337a55..d6f1109d1 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd_with_insecure_registry.golden b/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd_with_insecure_registry.golden index cc81fa5e5..f5a9ac81f 100644 --- a/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd_with_insecure_registry.golden +++ b/pkg/scripts/testdata/TestKubeadmCentOS-with_containerd_with_insecure_registry.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestKubeadmCoreOS-force.golden b/pkg/scripts/testdata/TestKubeadmFlatcar-force.golden similarity index 100% rename from pkg/scripts/testdata/TestKubeadmCoreOS-force.golden rename to pkg/scripts/testdata/TestKubeadmFlatcar-force.golden diff --git a/pkg/scripts/testdata/TestKubeadmCoreOS-overwrite_registry.golden b/pkg/scripts/testdata/TestKubeadmFlatcar-overwrite_registry.golden similarity index 100% rename from pkg/scripts/testdata/TestKubeadmCoreOS-overwrite_registry.golden rename to pkg/scripts/testdata/TestKubeadmFlatcar-overwrite_registry.golden diff --git a/pkg/scripts/testdata/TestKubeadmCoreOS-overwrite_registry_insecure.golden b/pkg/scripts/testdata/TestKubeadmFlatcar-overwrite_registry_insecure.golden similarity index 100% rename from pkg/scripts/testdata/TestKubeadmCoreOS-overwrite_registry_insecure.golden rename to pkg/scripts/testdata/TestKubeadmFlatcar-overwrite_registry_insecure.golden diff --git a/pkg/scripts/testdata/TestKubeadmCoreOS-simple.golden b/pkg/scripts/testdata/TestKubeadmFlatcar-simple.golden similarity index 100% rename from pkg/scripts/testdata/TestKubeadmCoreOS-simple.golden rename to pkg/scripts/testdata/TestKubeadmFlatcar-simple.golden diff --git a/pkg/scripts/testdata/TestRemoveBinariesCoreOS.golden b/pkg/scripts/testdata/TestRemoveBinariesFlatcar.golden similarity index 100% rename from pkg/scripts/testdata/TestRemoveBinariesCoreOS.golden rename to pkg/scripts/testdata/TestRemoveBinariesFlatcar.golden diff --git a/pkg/scripts/testdata/TestUpgradeKubeadmAndCNIAmazonLinux.golden b/pkg/scripts/testdata/TestUpgradeKubeadmAndCNIAmazonLinux.golden index d99c3c7d9..36e9732be 100644 --- a/pkg/scripts/testdata/TestUpgradeKubeadmAndCNIAmazonLinux.golden +++ b/pkg/scripts/testdata/TestUpgradeKubeadmAndCNIAmazonLinux.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestUpgradeKubeadmAndCNICentOS.golden b/pkg/scripts/testdata/TestUpgradeKubeadmAndCNICentOS.golden index d768d7037..c52d7bc9c 100644 --- a/pkg/scripts/testdata/TestUpgradeKubeadmAndCNICentOS.golden +++ b/pkg/scripts/testdata/TestUpgradeKubeadmAndCNICentOS.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestUpgradeKubeadmAndCNICoreOS.golden b/pkg/scripts/testdata/TestUpgradeKubeadmAndCNIFlatcar.golden similarity index 100% rename from pkg/scripts/testdata/TestUpgradeKubeadmAndCNICoreOS.golden rename to pkg/scripts/testdata/TestUpgradeKubeadmAndCNIFlatcar.golden diff --git a/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlAmazonLinux.golden b/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlAmazonLinux.golden index 570b7ce24..d12c127b4 100644 --- a/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlAmazonLinux.golden +++ b/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlAmazonLinux.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlCentOS.golden b/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlCentOS.golden index fff2d10b9..83c52ddc9 100644 --- a/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlCentOS.golden +++ b/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlCentOS.golden @@ -65,7 +65,8 @@ sudo yum install -y \ conntrack-tools \ ebtables \ socat \ - iproute-tc + iproute-tc \ + rsync diff --git a/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlCoreOS.golden b/pkg/scripts/testdata/TestUpgradeKubeletAndKubectlFlatcar.golden similarity index 100% rename from pkg/scripts/testdata/TestUpgradeKubeletAndKubectlCoreOS.golden rename to pkg/scripts/testdata/TestUpgradeKubeletAndKubectlFlatcar.golden diff --git a/pkg/tasks/kubernetes_binaries.go b/pkg/tasks/kubernetes_binaries.go index 448436f94..cb7b714bd 100644 --- a/pkg/tasks/kubernetes_binaries.go +++ b/pkg/tasks/kubernetes_binaries.go @@ -26,23 +26,23 @@ import ( func upgradeKubeletAndKubectlBinaries(s *state.State, node kubeoneapi.HostConfig) error { return runOnOS(s, node.OperatingSystem, map[kubeoneapi.OperatingSystemName]runOnOSFn{ - kubeoneapi.OperatingSystemNameUbuntu: upgradeKubeletAndKubectlBinariesDebian, - kubeoneapi.OperatingSystemNameCoreOS: upgradeKubeletAndKubectlBinariesCoreOS, - kubeoneapi.OperatingSystemNameFlatcar: upgradeKubeletAndKubectlBinariesCoreOS, - kubeoneapi.OperatingSystemNameCentOS: upgradeKubeletAndKubectlBinariesCentOS, kubeoneapi.OperatingSystemNameAmazon: upgradeKubeletAndKubectlBinariesAmazonLinux, + kubeoneapi.OperatingSystemNameCentOS: upgradeKubeletAndKubectlBinariesCentOS, + kubeoneapi.OperatingSystemNameDebian: upgradeKubeletAndKubectlBinariesDebian, + kubeoneapi.OperatingSystemNameFlatcar: upgradeKubeletAndKubectlBinariesFlatcar, kubeoneapi.OperatingSystemNameRHEL: upgradeKubeletAndKubectlBinariesCentOS, + kubeoneapi.OperatingSystemNameUbuntu: upgradeKubeletAndKubectlBinariesDebian, }) } func upgradeKubeadmAndCNIBinaries(s *state.State, node kubeoneapi.HostConfig) error { return runOnOS(s, node.OperatingSystem, map[kubeoneapi.OperatingSystemName]runOnOSFn{ - kubeoneapi.OperatingSystemNameUbuntu: upgradeKubeadmAndCNIBinariesDebian, - kubeoneapi.OperatingSystemNameCoreOS: upgradeKubeadmAndCNIBinariesCoreOS, - kubeoneapi.OperatingSystemNameFlatcar: upgradeKubeadmAndCNIBinariesCoreOS, - kubeoneapi.OperatingSystemNameCentOS: upgradeKubeadmAndCNIBinariesCentOS, kubeoneapi.OperatingSystemNameAmazon: upgradeKubeadmAndCNIBinariesAmazonLinux, + kubeoneapi.OperatingSystemNameCentOS: upgradeKubeadmAndCNIBinariesCentOS, + kubeoneapi.OperatingSystemNameDebian: upgradeKubeadmAndCNIBinariesDebian, + kubeoneapi.OperatingSystemNameFlatcar: upgradeKubeadmAndCNIBinariesFlatcar, kubeoneapi.OperatingSystemNameRHEL: upgradeKubeadmAndCNIBinariesCentOS, + kubeoneapi.OperatingSystemNameUbuntu: upgradeKubeadmAndCNIBinariesDebian, }) } @@ -57,8 +57,8 @@ func upgradeKubeletAndKubectlBinariesDebian(s *state.State) error { return errors.WithStack(err) } -func upgradeKubeletAndKubectlBinariesCoreOS(s *state.State) error { - cmd, err := scripts.UpgradeKubeletAndKubectlCoreOS(s.Cluster.Versions.Kubernetes) +func upgradeKubeletAndKubectlBinariesFlatcar(s *state.State) error { + cmd, err := scripts.UpgradeKubeletAndKubectlFlatcar(s.Cluster.Versions.Kubernetes) if err != nil { return err } @@ -123,8 +123,8 @@ func upgradeKubeadmAndCNIBinariesAmazonLinux(s *state.State) error { return errors.WithStack(err) } -func upgradeKubeadmAndCNIBinariesCoreOS(s *state.State) error { - cmd, err := scripts.UpgradeKubeadmAndCNICoreOS(s.Cluster.Versions.Kubernetes) +func upgradeKubeadmAndCNIBinariesFlatcar(s *state.State) error { + cmd, err := scripts.UpgradeKubeadmAndCNIFlatcar(s.Cluster.Versions.Kubernetes) if err != nil { return err } diff --git a/pkg/tasks/nodes.go b/pkg/tasks/nodes.go index a827dcc3e..5b7fc8752 100644 --- a/pkg/tasks/nodes.go +++ b/pkg/tasks/nodes.go @@ -56,3 +56,34 @@ func uncordonNode(s *state.State, host kubeoneapi.HostConfig) error { return errors.WithStack(updateErr) } + +func restartKubeAPIServer(s *state.State) error { + s.Logger.Infoln("Restarting unhealthy API servers if needed...") + + return s.RunTaskOnControlPlane(func(s *state.State, node *kubeoneapi.HostConfig, _ ssh.Connection) error { + return restartKubeAPIServerOnOS(s, *node) + }, state.RunSequentially) +} + +func restartKubeAPIServerOnOS(s *state.State, node kubeoneapi.HostConfig) error { + return runOnOS(s, node.OperatingSystem, map[kubeoneapi.OperatingSystemName]runOnOSFn{ + kubeoneapi.OperatingSystemNameAmazon: restartKubeAPIServerCrictl, + kubeoneapi.OperatingSystemNameCentOS: restartKubeAPIServerCrictl, + kubeoneapi.OperatingSystemNameDebian: restartKubeAPIServerCrictl, + kubeoneapi.OperatingSystemNameFlatcar: restartKubeAPIServerDocker, + kubeoneapi.OperatingSystemNameRHEL: restartKubeAPIServerCrictl, + kubeoneapi.OperatingSystemNameUbuntu: restartKubeAPIServerCrictl, + }) +} + +func restartKubeAPIServerCrictl(s *state.State) error { + _, _, err := s.Runner.RunRaw(scripts.RestartKubeAPIServerCrictl()) + + return errors.WithStack(err) +} + +func restartKubeAPIServerDocker(s *state.State) error { + _, _, err := s.Runner.RunRaw(scripts.RestartKubeAPIServerDocker()) + + return errors.WithStack(err) +} diff --git a/pkg/tasks/prerequisites.go b/pkg/tasks/prerequisites.go index a6763b580..ff0c2d342 100644 --- a/pkg/tasks/prerequisites.go +++ b/pkg/tasks/prerequisites.go @@ -85,12 +85,12 @@ func createEnvironmentFile(s *state.State) error { func installKubeadm(s *state.State, node kubeoneapi.HostConfig) error { return runOnOS(s, node.OperatingSystem, map[kubeoneapi.OperatingSystemName]runOnOSFn{ - kubeoneapi.OperatingSystemNameUbuntu: installKubeadmDebian, - kubeoneapi.OperatingSystemNameCoreOS: installKubeadmCoreOS, kubeoneapi.OperatingSystemNameAmazon: installKubeadmAmazonLinux, - kubeoneapi.OperatingSystemNameFlatcar: installKubeadmCoreOS, kubeoneapi.OperatingSystemNameCentOS: installKubeadmCentOS, + kubeoneapi.OperatingSystemNameDebian: installKubeadmDebian, + kubeoneapi.OperatingSystemNameFlatcar: installKubeadmFlatcar, kubeoneapi.OperatingSystemNameRHEL: installKubeadmCentOS, + kubeoneapi.OperatingSystemNameUbuntu: installKubeadmDebian, }) } @@ -127,8 +127,8 @@ func installKubeadmAmazonLinux(s *state.State) error { return errors.WithStack(err) } -func installKubeadmCoreOS(s *state.State) error { - cmd, err := scripts.KubeadmCoreOS(s.Cluster) +func installKubeadmFlatcar(s *state.State) error { + cmd, err := scripts.KubeadmFlatcar(s.Cluster) if err != nil { return err } diff --git a/pkg/tasks/probes.go b/pkg/tasks/probes.go index 3a1661586..ebe96a192 100644 --- a/pkg/tasks/probes.go +++ b/pkg/tasks/probes.go @@ -110,8 +110,7 @@ func runProbes(s *state.State) error { } for _, host := range s.Cluster.ControlPlane.Hosts { - switch host.OperatingSystem { - case kubeoneapi.OperatingSystemNameFlatcar, kubeoneapi.OperatingSystemNameCoreOS: + if host.OperatingSystem == kubeoneapi.OperatingSystemNameFlatcar { s.Cluster.ContainerRuntime.Docker = &kubeoneapi.ContainerRuntimeDocker{} } } @@ -189,8 +188,7 @@ func investigateHost(s *state.State, node *kubeoneapi.HostConfig, conn ssh.Conne containerRuntimeOpts := []systemdUnitInfoOpt{withComponentVersion(versionCmdGenerator)} - switch h.Config.OperatingSystem { - case kubeoneapi.OperatingSystemNameCoreOS, kubeoneapi.OperatingSystemNameFlatcar: + if h.Config.OperatingSystem == kubeoneapi.OperatingSystemNameFlatcar { // Flatcar is special containerRuntimeOpts = []systemdUnitInfoOpt{withFlatcarContainerRuntimeVersion} } diff --git a/pkg/tasks/reset.go b/pkg/tasks/reset.go index 0b807ef19..2685d9016 100644 --- a/pkg/tasks/reset.go +++ b/pkg/tasks/reset.go @@ -129,12 +129,12 @@ func removeBinaries(s *state.State, node *kubeoneapi.HostConfig, conn ssh.Connec } return runOnOS(s, node.OperatingSystem, map[kubeoneapi.OperatingSystemName]runOnOSFn{ - kubeoneapi.OperatingSystemNameUbuntu: removeBinariesDebian, - kubeoneapi.OperatingSystemNameCoreOS: removeBinariesCoreOS, - kubeoneapi.OperatingSystemNameFlatcar: removeBinariesCoreOS, + kubeoneapi.OperatingSystemNameAmazon: removeBinariesAmazonLinux, kubeoneapi.OperatingSystemNameCentOS: removeBinariesCentOS, + kubeoneapi.OperatingSystemNameDebian: removeBinariesDebian, + kubeoneapi.OperatingSystemNameFlatcar: removeBinariesFlatcar, kubeoneapi.OperatingSystemNameRHEL: removeBinariesCentOS, - kubeoneapi.OperatingSystemNameAmazon: removeBinariesAmazonLinux, + kubeoneapi.OperatingSystemNameUbuntu: removeBinariesDebian, }) } @@ -168,8 +168,8 @@ func removeBinariesAmazonLinux(s *state.State) error { return errors.WithStack(err) } -func removeBinariesCoreOS(s *state.State) error { - cmd, err := scripts.RemoveBinariesCoreOS() +func removeBinariesFlatcar(s *state.State) error { + cmd, err := scripts.RemoveBinariesFlatcar() if err != nil { return err } diff --git a/pkg/tasks/tasks.go b/pkg/tasks/tasks.go index 88e1d8a80..6768c29a0 100644 --- a/pkg/tasks/tasks.go +++ b/pkg/tasks/tasks.go @@ -114,6 +114,7 @@ func WithFullInstall(t Tasks) Tasks { {Fn: repairClusterIfNeeded, ErrMsg: "failed to repair cluster"}, {Fn: joinControlplaneNode, ErrMsg: "failed to join other masters a cluster"}, {Fn: saveKubeconfig, ErrMsg: "failed to save kubeconfig to the local machine"}, + {Fn: restartKubeAPIServer, ErrMsg: "failed to restart unhealthy kube-apiserver"}, }...). append(kubernetesResources()...). append( @@ -188,6 +189,7 @@ func WithUpgrade(t Tasks) Tasks { }...). append(kubernetesResources()...). append( + Task{Fn: restartKubeAPIServer, ErrMsg: "failed to restart unhealthy kube-apiserver"}, Task{Fn: upgradeStaticWorkers, ErrMsg: "unable to upgrade static worker nodes"}, Task{ Fn: upgradeMachineDeployments, diff --git a/pkg/templates/externalccm/openstack.go b/pkg/templates/externalccm/openstack.go index 88741a67f..1f862bde0 100644 --- a/pkg/templates/externalccm/openstack.go +++ b/pkg/templates/externalccm/openstack.go @@ -164,8 +164,7 @@ func osDaemonSet(image string) *appsv1.DaemonSet { runAsUser int64 = 1001 hostPathDirectoryOrCreate = corev1.HostPathDirectoryOrCreate - caCertsPath = "/etc/ssl/certs" - flexvolumeDir = "/usr/libexec/kubernetes/kubelet-plugins/volume/exec" + caCertsPath = "/etc/ssl/certs" ) return &appsv1.DaemonSet{ @@ -235,10 +234,6 @@ func osDaemonSet(image string) *appsv1.DaemonSet { MountPath: "/etc/config", ReadOnly: true, }, - { - Name: "flexvolume-dir", - MountPath: "/usr/libexec/kubernetes/kubelet-plugins/volume/exec", - }, }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ @@ -249,15 +244,6 @@ func osDaemonSet(image string) *appsv1.DaemonSet { }, HostNetwork: true, Volumes: []corev1.Volume{ - { - Name: "flexvolume-dir", - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: flexvolumeDir, - Type: &hostPathDirectoryOrCreate, - }, - }, - }, { Name: "k8s-certs", VolumeSource: corev1.VolumeSource{ diff --git a/pkg/templates/machinecontroller/deployment.go b/pkg/templates/machinecontroller/deployment.go index ecd06b8a8..480f0a8ea 100644 --- a/pkg/templates/machinecontroller/deployment.go +++ b/pkg/templates/machinecontroller/deployment.go @@ -48,7 +48,7 @@ const ( MachineControllerAppLabelValue = "machine-controller" MachineControllerImageRegistry = "docker.io" MachineControllerImage = "/kubermatic/machine-controller:" - MachineControllerTag = "v1.23.1" + MachineControllerTag = "v1.25.0" ) func CRDs() []runtime.Object { @@ -760,6 +760,13 @@ func machineControllerDeployment(cluster *kubeoneapi.KubeOneCluster, credentials "-node-csr-approver", } + switch { + case cluster.ContainerRuntime.Containerd != nil: + fallthrough + case cluster.ContainerRuntime.Docker != nil: + args = append(args, "-node-container-runtime", cluster.ContainerRuntime.String()) + } + if cluster.Proxy.HTTP != "" { args = append(args, "-node-http-proxy", cluster.Proxy.HTTP) } diff --git a/test/e2e/os.go b/test/e2e/os.go index 5e757043e..d4b73101f 100644 --- a/test/e2e/os.go +++ b/test/e2e/os.go @@ -29,7 +29,6 @@ const ( OperatingSystemUbuntu OperatingSystem = "ubuntu" OperatingSystemCentOS7 OperatingSystem = "centos7" OperatingSystemCentOS8 OperatingSystem = "centos" - OperatingSystemCoreOS OperatingSystem = "coreos" OperatingSystemFlatcar OperatingSystem = "flatcar" OperatingSystemDefault OperatingSystem = "" ) @@ -40,8 +39,11 @@ const ( func ValidateOperatingSystem(osName string) error { switch OperatingSystem(osName) { - case OperatingSystemUbuntu, OperatingSystemCoreOS, OperatingSystemFlatcar, - OperatingSystemCentOS7, OperatingSystemCentOS8, OperatingSystemDefault: + case OperatingSystemUbuntu, + OperatingSystemFlatcar, + OperatingSystemCentOS7, + OperatingSystemCentOS8, + OperatingSystemDefault: return nil } return errors.New("failed to validate operating system") @@ -79,7 +81,7 @@ func sshUsername(osName OperatingSystem) (string, error) { return "ubuntu", nil case OperatingSystemCentOS7, OperatingSystemCentOS8: return "centos", nil - case OperatingSystemCoreOS, OperatingSystemFlatcar: + case OperatingSystemFlatcar: return "core", nil }