From 76c1e7959859f8c8bf5a35a299962806ea8f5549 Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Wed, 27 Oct 2021 02:36:41 +0100 Subject: [PATCH 1/2] support k8s < v1.19 & watch-ingress-without-class --- .../addons/ingress/ingress-deploy.yaml.tmpl | 4 +-- pkg/addons/addons.go | 25 ++++++++----------- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/deploy/addons/ingress/ingress-deploy.yaml.tmpl b/deploy/addons/ingress/ingress-deploy.yaml.tmpl index 3b7fd212558b..784100cc1ad1 100644 --- a/deploy/addons/ingress/ingress-deploy.yaml.tmpl +++ b/deploy/addons/ingress/ingress-deploy.yaml.tmpl @@ -365,12 +365,12 @@ spec: - --election-id=ingress-controller-leader {{- if eq .IngressAPIVersion "v1"}} - --controller-class=k8s.io/ingress-nginx + - --watch-ingress-without-class=true {{- end}} {{- if eq .IngressAPIVersion "v1beta1"}} - --ingress-class=nginx - - --watch-ingress-without-class=true - - --publish-status-address=localhost {{- end}} + - --publish-status-address=localhost - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller - --report-node-internal-ip-address - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index 6e1428aa7fc4..82add276fff7 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -179,6 +179,13 @@ func EnableOrDisableAddon(cc *config.ClusterConfig, name string, val string) err exit.Error(reason.GuestCpConfig, "Error getting primary control plane", err) } + // maintain backwards compatibility for ingress with k8s < v1.19 + if strings.HasPrefix(name, "ingress") && enable { + if err := supportLegacyIngress(addon, *cc); err != nil { + return err + } + } + // Persist images even if the machine is running so starting gets the correct images. images, customRegistries, err := assets.SelectAndPersistImages(addon, cc) if err != nil { @@ -229,9 +236,6 @@ func addonSpecificChecks(cc *config.ClusterConfig, name string, enable bool, run out.Styled(style.Tip, `After the addon is enabled, please run "minikube tunnel" and your ingress resources would be available at "127.0.0.1"`) } } - if err := supportLegacyIngress(cc); err != nil { - return false, err - } } if strings.HasPrefix(name, "istio") && enable { @@ -290,29 +294,20 @@ func isAddonAlreadySet(cc *config.ClusterConfig, addon *assets.Addon, enable boo return false } -// maintain backwards compatibility with k8s < v1.19 -// by replacing images with old versions if custom ones are not already provided -func supportLegacyIngress(cc *config.ClusterConfig) error { +// maintain backwards compatibility for ingress with k8s < v1.19 by replacing default addon images with older versions +func supportLegacyIngress(addon *assets.Addon, cc config.ClusterConfig) error { v, err := util.ParseKubernetesVersion(cc.KubernetesConfig.KubernetesVersion) if err != nil { return errors.Wrap(err, "parsing Kubernetes version") } if semver.MustParseRange("<1.19.0")(v) { - imgs := map[string]string{ + addon.Images = map[string]string{ // https://github.com/kubernetes/ingress-nginx/blob/0a2ec01eb4ec0e1b29c4b96eb838a2e7bfe0e9f6/deploy/static/provider/kind/deploy.yaml#L328 "IngressController": "ingress-nginx/controller:v0.49.3@sha256:35fe394c82164efa8f47f3ed0be981b3f23da77175bbb8268a9ae438851c8324", // issues: https://github.com/kubernetes/ingress-nginx/issues/7418 and https://github.com/jet/kube-webhook-certgen/issues/30 "KubeWebhookCertgenCreate": "docker.io/jettech/kube-webhook-certgen:v1.5.1@sha256:950833e19ade18cd389d647efb88992a7cc077abedef343fa59e012d376d79b7", "KubeWebhookCertgenPatch": "docker.io/jettech/kube-webhook-certgen:v1.5.1@sha256:950833e19ade18cd389d647efb88992a7cc077abedef343fa59e012d376d79b7", } - if cc.CustomAddonImages == nil { - cc.CustomAddonImages = map[string]string{} - } - for name, path := range imgs { - if _, exists := cc.CustomAddonImages[name]; !exists { - cc.CustomAddonImages[name] = path - } - } } return nil } From ea3aa4f3a85fcb3c47b6bae6243207cca1d8d988 Mon Sep 17 00:00:00 2001 From: Predrag Rogic Date: Thu, 28 Oct 2021 03:50:06 +0100 Subject: [PATCH 2/2] support ingress-dns addon, add tests --- pkg/addons/addons.go | 31 +++++-- test/integration/addons_test.go | 18 +++- test/integration/ingress_addon_legacy_test.go | 83 ++++++++++++++++++ ...ample.yaml => ingress-dns-example-v1.yaml} | 0 .../testdata/ingress-dns-example-v1beta1.yaml | 85 +++++++++++++++++++ ...nginx-ingv1.yaml => nginx-ingress-v1.yaml} | 3 - .../testdata/nginx-ingress-v1beta1.yaml | 19 +++++ 7 files changed, 226 insertions(+), 13 deletions(-) create mode 100644 test/integration/ingress_addon_legacy_test.go rename test/integration/testdata/{ingress-dns-example.yaml => ingress-dns-example-v1.yaml} (100%) create mode 100644 test/integration/testdata/ingress-dns-example-v1beta1.yaml rename test/integration/testdata/{nginx-ingv1.yaml => nginx-ingress-v1.yaml} (78%) create mode 100644 test/integration/testdata/nginx-ingress-v1beta1.yaml diff --git a/pkg/addons/addons.go b/pkg/addons/addons.go index 82add276fff7..80779a6cdcb0 100644 --- a/pkg/addons/addons.go +++ b/pkg/addons/addons.go @@ -179,7 +179,7 @@ func EnableOrDisableAddon(cc *config.ClusterConfig, name string, val string) err exit.Error(reason.GuestCpConfig, "Error getting primary control plane", err) } - // maintain backwards compatibility for ingress with k8s < v1.19 + // maintain backwards compatibility for ingress and ingress-dns addons with k8s < v1.19 if strings.HasPrefix(name, "ingress") && enable { if err := supportLegacyIngress(addon, *cc); err != nil { return err @@ -294,21 +294,36 @@ func isAddonAlreadySet(cc *config.ClusterConfig, addon *assets.Addon, enable boo return false } -// maintain backwards compatibility for ingress with k8s < v1.19 by replacing default addon images with older versions +// maintain backwards compatibility for ingress and ingress-dns addons with k8s < v1.19 by replacing default addons' images with compatible versions func supportLegacyIngress(addon *assets.Addon, cc config.ClusterConfig) error { v, err := util.ParseKubernetesVersion(cc.KubernetesConfig.KubernetesVersion) if err != nil { return errors.Wrap(err, "parsing Kubernetes version") } if semver.MustParseRange("<1.19.0")(v) { - addon.Images = map[string]string{ - // https://github.com/kubernetes/ingress-nginx/blob/0a2ec01eb4ec0e1b29c4b96eb838a2e7bfe0e9f6/deploy/static/provider/kind/deploy.yaml#L328 - "IngressController": "ingress-nginx/controller:v0.49.3@sha256:35fe394c82164efa8f47f3ed0be981b3f23da77175bbb8268a9ae438851c8324", - // issues: https://github.com/kubernetes/ingress-nginx/issues/7418 and https://github.com/jet/kube-webhook-certgen/issues/30 - "KubeWebhookCertgenCreate": "docker.io/jettech/kube-webhook-certgen:v1.5.1@sha256:950833e19ade18cd389d647efb88992a7cc077abedef343fa59e012d376d79b7", - "KubeWebhookCertgenPatch": "docker.io/jettech/kube-webhook-certgen:v1.5.1@sha256:950833e19ade18cd389d647efb88992a7cc077abedef343fa59e012d376d79b7", + if addon.Name() == "ingress" { + addon.Images = map[string]string{ + // https://github.com/kubernetes/ingress-nginx/blob/0a2ec01eb4ec0e1b29c4b96eb838a2e7bfe0e9f6/deploy/static/provider/kind/deploy.yaml#L328 + "IngressController": "ingress-nginx/controller:v0.49.3@sha256:35fe394c82164efa8f47f3ed0be981b3f23da77175bbb8268a9ae438851c8324", + // issues: https://github.com/kubernetes/ingress-nginx/issues/7418 and https://github.com/jet/kube-webhook-certgen/issues/30 + "KubeWebhookCertgenCreate": "docker.io/jettech/kube-webhook-certgen:v1.5.1@sha256:950833e19ade18cd389d647efb88992a7cc077abedef343fa59e012d376d79b7", + "KubeWebhookCertgenPatch": "docker.io/jettech/kube-webhook-certgen:v1.5.1@sha256:950833e19ade18cd389d647efb88992a7cc077abedef343fa59e012d376d79b7", + } + addon.Registries = map[string]string{ + "IngressController": "k8s.gcr.io", + } + return nil } + if addon.Name() == "ingress-dns" { + addon.Images = map[string]string{ + "IngressDNS": "cryptexlabs/minikube-ingress-dns:0.3.0@sha256:e252d2a4c704027342b303cc563e95d2e71d2a0f1404f55d676390e28d5093ab", + } + addon.Registries = nil + return nil + } + return fmt.Errorf("supportLegacyIngress called for unexpected addon %q - nothing to do here", addon.Name()) } + return nil } diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index cba3456789a7..b3983ecb5d99 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -34,6 +34,7 @@ import ( "testing" "time" + "github.com/blang/semver/v4" retryablehttp "github.com/hashicorp/go-retryablehttp" "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/minikube/detect" @@ -163,10 +164,23 @@ func validateIngressAddon(ctx context.Context, t *testing.T, profile string) { t.Fatalf("failed waiting for ingress-nginx-controller : %v", err) } + // use nginx ingress yaml that corresponds to k8s version + // default: k8s >= v1.19, ingress api v1 + ingressYaml := "nginx-ingress-v1.yaml" + ingressDNSYaml := "ingress-dns-example-v1.yaml" + v, err := client.ServerVersion() + if err != nil { + t.Log("failed to get k8s version, assuming v1.19+ => ingress api v1") + } else if semver.MustParseRange("<1.19.0")(semver.MustParse(fmt.Sprintf("%s.%s.0", v.Major, v.Minor))) { + // legacy: k8s < v1.19 & ingress api v1beta1 + ingressYaml = "nginx-ingress-v1beta1.yaml" + ingressDNSYaml = "ingress-dns-example-v1beta1.yaml" + } + // create networking.k8s.io/v1 ingress createv1Ingress := func() error { // apply networking.k8s.io/v1 ingress - rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "nginx-ingv1.yaml"))) + rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, ingressYaml))) if err != nil { return err } @@ -220,7 +234,7 @@ func validateIngressAddon(ctx context.Context, t *testing.T, profile string) { } // check the ingress-dns addon here as well - rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "ingress-dns-example.yaml"))) + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, ingressDNSYaml))) if err != nil { t.Errorf("failed to kubectl replace ingress-dns-example. args %q. %v", rr.Command(), err) } diff --git a/test/integration/ingress_addon_legacy_test.go b/test/integration/ingress_addon_legacy_test.go new file mode 100644 index 000000000000..ba8a15dcb7b1 --- /dev/null +++ b/test/integration/ingress_addon_legacy_test.go @@ -0,0 +1,83 @@ +//go:build integration +// +build integration + +/* +Copyright 2021 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "context" + "os/exec" + "testing" +) + +// TestIngressAddonLegacy tests ingress and ingress-dns addons with legacy k8s version <1.19 +func TestIngressAddonLegacy(t *testing.T) { + if NoneDriver() { + t.Skipf("skipping: none driver does not support ingress") + } + + profile := UniqueProfileName("ingress-addon-legacy") + ctx, cancel := context.WithTimeout(context.Background(), Minutes(10)) + defer Cleanup(t, profile, cancel) + + t.Run("StartLegacyK8sCluster", func(t *testing.T) { + args := append([]string{"start", "-p", profile, "--kubernetes-version=v1.18.20", "--memory=4096", "--wait=true", "--alsologtostderr", "-v=5"}, StartArgs()...) + rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) + if err != nil { + t.Errorf("failed to start minikube with args: %q : %v", rr.Command(), err) + } + }) + + t.Run("serial", func(t *testing.T) { + tests := []struct { + name string + validator validateFunc + }{ + {"ValidateIngressAddonActivation", validateIngressAddonActivation}, + {"ValidateIngressDNSAddonActivation", validateIngressDNSAddonActivation}, + {"ValidateIngressAddons", validateIngressAddon}, + } + for _, tc := range tests { + tc := tc + if ctx.Err() == context.DeadlineExceeded { + t.Fatalf("Unable to run more tests (deadline exceeded)") + } + t.Run(tc.name, func(t *testing.T) { + tc.validator(ctx, t, profile) + }) + } + }) +} + +// validateIngressAddonActivation tests ingress addon activation +func validateIngressAddonActivation(ctx context.Context, t *testing.T, profile string) { + defer PostMortemLogs(t, profile) + + if _, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "enable", "ingress", "--alsologtostderr", "-v=5")); err != nil { + t.Errorf("failed to enable ingress addon: %v", err) + } +} + +// validateIngressDNSAddonActivation tests ingress-dns addon activation +func validateIngressDNSAddonActivation(ctx context.Context, t *testing.T, profile string) { + defer PostMortemLogs(t, profile) + + if _, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "enable", "ingress-dns", "--alsologtostderr", "-v=5")); err != nil { + t.Errorf("failed to enable ingress-dns addon: %v", err) + } +} diff --git a/test/integration/testdata/ingress-dns-example.yaml b/test/integration/testdata/ingress-dns-example-v1.yaml similarity index 100% rename from test/integration/testdata/ingress-dns-example.yaml rename to test/integration/testdata/ingress-dns-example-v1.yaml diff --git a/test/integration/testdata/ingress-dns-example-v1beta1.yaml b/test/integration/testdata/ingress-dns-example-v1beta1.yaml new file mode 100644 index 000000000000..09f2817df4a0 --- /dev/null +++ b/test/integration/testdata/ingress-dns-example-v1beta1.yaml @@ -0,0 +1,85 @@ +# Copyright 2021 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hello-world-app + namespace: default +spec: + selector: + matchLabels: + app: hello-world-app + template: + metadata: + labels: + app: hello-world-app + spec: + containers: + - name: hello-world-app + image: gcr.io/google-samples/hello-app:1.0 + ports: + - containerPort: 8080 +--- +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: example-ingress + namespace: kube-system + annotations: + # needed for k8s < v1.18: https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#deprecating-the-ingress-class-annotation + kubernetes.io/ingress.class: nginx +spec: + rules: + - host: hello-john.test + http: + paths: + - path: / + pathType: Prefix + backend: + serviceName: hello-world-app + servicePort: 80 + - host: hello-jane.test + http: + paths: + - path: / + pathType: Prefix + backend: + serviceName: hello-world-app + servicePort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: hello-world-app + namespace: kube-system +spec: + type: ExternalName + externalName: hello-world-app.default.svc.cluster.local +--- +apiVersion: v1 +kind: Service +metadata: + name: hello-world-app + namespace: default +spec: + ports: + - name: http + port: 80 + targetPort: 8080 + protocol: TCP + type: NodePort + selector: + app: hello-world-app diff --git a/test/integration/testdata/nginx-ingv1.yaml b/test/integration/testdata/nginx-ingress-v1.yaml similarity index 78% rename from test/integration/testdata/nginx-ingv1.yaml rename to test/integration/testdata/nginx-ingress-v1.yaml index e95cf7fc029e..e5c6766b0518 100644 --- a/test/integration/testdata/nginx-ingv1.yaml +++ b/test/integration/testdata/nginx-ingress-v1.yaml @@ -2,9 +2,6 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: nginx-ingress - annotations: - # use the shared ingress-nginx - kubernetes.io/ingress.class: nginx labels: integration-test: ingress spec: diff --git a/test/integration/testdata/nginx-ingress-v1beta1.yaml b/test/integration/testdata/nginx-ingress-v1beta1.yaml new file mode 100644 index 000000000000..5e3fcc2350e4 --- /dev/null +++ b/test/integration/testdata/nginx-ingress-v1beta1.yaml @@ -0,0 +1,19 @@ +apiVersion: networking.k8s.io/v1beta1 +kind: Ingress +metadata: + name: nginx-ingress + annotations: + # needed for k8s < v1.18: https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#deprecating-the-ingress-class-annotation + kubernetes.io/ingress.class: nginx + labels: + integration-test: ingress +spec: + rules: + - host: nginx.example.com + http: + paths: + - path: / + pathType: Prefix + backend: + serviceName: nginx + servicePort: 80