Skip to content

Commit

Permalink
Merge pull request #12794 from prezha/improve-ingress
Browse files Browse the repository at this point in the history
support k8s < v1.19 & watch-ingress-without-class
  • Loading branch information
sharifelgamal committed Oct 28, 2021
2 parents 2aa49b6 + ea3aa4f commit 6bbf98d
Show file tree
Hide file tree
Showing 8 changed files with 233 additions and 25 deletions.
4 changes: 2 additions & 2 deletions deploy/addons/ingress/ingress-deploy.yaml.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -365,12 +365,12 @@ spec:
- --election-id=ingress-controller-leader
{{- if eq .IngressAPIVersion "v1"}}
- --controller-class=k8s.io/ingress-nginx
- --watch-ingress-without-class=true
{{- end}}
{{- if eq .IngressAPIVersion "v1beta1"}}
- --ingress-class=nginx
- --watch-ingress-without-class=true
- --publish-status-address=localhost
{{- end}}
- --publish-status-address=localhost
- --configmap=$(POD_NAMESPACE)/ingress-nginx-controller
- --report-node-internal-ip-address
- --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
Expand Down
46 changes: 28 additions & 18 deletions pkg/addons/addons.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,13 @@ func EnableOrDisableAddon(cc *config.ClusterConfig, name string, val string) err
exit.Error(reason.GuestCpConfig, "Error getting primary control plane", err)
}

// maintain backwards compatibility for ingress and ingress-dns addons with k8s < v1.19
if strings.HasPrefix(name, "ingress") && enable {
if err := supportLegacyIngress(addon, *cc); err != nil {
return err
}
}

// Persist images even if the machine is running so starting gets the correct images.
images, customRegistries, err := assets.SelectAndPersistImages(addon, cc)
if err != nil {
Expand Down Expand Up @@ -229,9 +236,6 @@ func addonSpecificChecks(cc *config.ClusterConfig, name string, enable bool, run
out.Styled(style.Tip, `After the addon is enabled, please run "minikube tunnel" and your ingress resources would be available at "127.0.0.1"`)
}
}
if err := supportLegacyIngress(cc); err != nil {
return false, err
}
}

if strings.HasPrefix(name, "istio") && enable {
Expand Down Expand Up @@ -290,30 +294,36 @@ func isAddonAlreadySet(cc *config.ClusterConfig, addon *assets.Addon, enable boo
return false
}

// maintain backwards compatibility with k8s < v1.19
// by replacing images with old versions if custom ones are not already provided
func supportLegacyIngress(cc *config.ClusterConfig) error {
// maintain backwards compatibility for ingress and ingress-dns addons with k8s < v1.19 by replacing default addons' images with compatible versions
func supportLegacyIngress(addon *assets.Addon, cc config.ClusterConfig) error {
v, err := util.ParseKubernetesVersion(cc.KubernetesConfig.KubernetesVersion)
if err != nil {
return errors.Wrap(err, "parsing Kubernetes version")
}
if semver.MustParseRange("<1.19.0")(v) {
imgs := map[string]string{
// https://github.com/kubernetes/ingress-nginx/blob/0a2ec01eb4ec0e1b29c4b96eb838a2e7bfe0e9f6/deploy/static/provider/kind/deploy.yaml#L328
"IngressController": "ingress-nginx/controller:v0.49.3@sha256:35fe394c82164efa8f47f3ed0be981b3f23da77175bbb8268a9ae438851c8324",
// issues: https://github.com/kubernetes/ingress-nginx/issues/7418 and https://github.com/jet/kube-webhook-certgen/issues/30
"KubeWebhookCertgenCreate": "docker.io/jettech/kube-webhook-certgen:v1.5.1@sha256:950833e19ade18cd389d647efb88992a7cc077abedef343fa59e012d376d79b7",
"KubeWebhookCertgenPatch": "docker.io/jettech/kube-webhook-certgen:v1.5.1@sha256:950833e19ade18cd389d647efb88992a7cc077abedef343fa59e012d376d79b7",
}
if cc.CustomAddonImages == nil {
cc.CustomAddonImages = map[string]string{}
if addon.Name() == "ingress" {
addon.Images = map[string]string{
// https://github.com/kubernetes/ingress-nginx/blob/0a2ec01eb4ec0e1b29c4b96eb838a2e7bfe0e9f6/deploy/static/provider/kind/deploy.yaml#L328
"IngressController": "ingress-nginx/controller:v0.49.3@sha256:35fe394c82164efa8f47f3ed0be981b3f23da77175bbb8268a9ae438851c8324",
// issues: https://github.com/kubernetes/ingress-nginx/issues/7418 and https://github.com/jet/kube-webhook-certgen/issues/30
"KubeWebhookCertgenCreate": "docker.io/jettech/kube-webhook-certgen:v1.5.1@sha256:950833e19ade18cd389d647efb88992a7cc077abedef343fa59e012d376d79b7",
"KubeWebhookCertgenPatch": "docker.io/jettech/kube-webhook-certgen:v1.5.1@sha256:950833e19ade18cd389d647efb88992a7cc077abedef343fa59e012d376d79b7",
}
addon.Registries = map[string]string{
"IngressController": "k8s.gcr.io",
}
return nil
}
for name, path := range imgs {
if _, exists := cc.CustomAddonImages[name]; !exists {
cc.CustomAddonImages[name] = path
if addon.Name() == "ingress-dns" {
addon.Images = map[string]string{
"IngressDNS": "cryptexlabs/minikube-ingress-dns:0.3.0@sha256:e252d2a4c704027342b303cc563e95d2e71d2a0f1404f55d676390e28d5093ab",
}
addon.Registries = nil
return nil
}
return fmt.Errorf("supportLegacyIngress called for unexpected addon %q - nothing to do here", addon.Name())
}

return nil
}

Expand Down
18 changes: 16 additions & 2 deletions test/integration/addons_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import (
"testing"
"time"

"github.com/blang/semver/v4"
retryablehttp "github.com/hashicorp/go-retryablehttp"
"k8s.io/minikube/pkg/kapi"
"k8s.io/minikube/pkg/minikube/detect"
Expand Down Expand Up @@ -163,10 +164,23 @@ func validateIngressAddon(ctx context.Context, t *testing.T, profile string) {
t.Fatalf("failed waiting for ingress-nginx-controller : %v", err)
}

// use nginx ingress yaml that corresponds to k8s version
// default: k8s >= v1.19, ingress api v1
ingressYaml := "nginx-ingress-v1.yaml"
ingressDNSYaml := "ingress-dns-example-v1.yaml"
v, err := client.ServerVersion()
if err != nil {
t.Log("failed to get k8s version, assuming v1.19+ => ingress api v1")
} else if semver.MustParseRange("<1.19.0")(semver.MustParse(fmt.Sprintf("%s.%s.0", v.Major, v.Minor))) {
// legacy: k8s < v1.19 & ingress api v1beta1
ingressYaml = "nginx-ingress-v1beta1.yaml"
ingressDNSYaml = "ingress-dns-example-v1beta1.yaml"
}

// create networking.k8s.io/v1 ingress
createv1Ingress := func() error {
// apply networking.k8s.io/v1 ingress
rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "nginx-ingv1.yaml")))
rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, ingressYaml)))
if err != nil {
return err
}
Expand Down Expand Up @@ -220,7 +234,7 @@ func validateIngressAddon(ctx context.Context, t *testing.T, profile string) {
}

// check the ingress-dns addon here as well
rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "ingress-dns-example.yaml")))
rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, ingressDNSYaml)))
if err != nil {
t.Errorf("failed to kubectl replace ingress-dns-example. args %q. %v", rr.Command(), err)
}
Expand Down
83 changes: 83 additions & 0 deletions test/integration/ingress_addon_legacy_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
//go:build integration
// +build integration

/*
Copyright 2021 The Kubernetes Authors All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
"context"
"os/exec"
"testing"
)

// TestIngressAddonLegacy tests ingress and ingress-dns addons with legacy k8s version <1.19
func TestIngressAddonLegacy(t *testing.T) {
if NoneDriver() {
t.Skipf("skipping: none driver does not support ingress")
}

profile := UniqueProfileName("ingress-addon-legacy")
ctx, cancel := context.WithTimeout(context.Background(), Minutes(10))
defer Cleanup(t, profile, cancel)

t.Run("StartLegacyK8sCluster", func(t *testing.T) {
args := append([]string{"start", "-p", profile, "--kubernetes-version=v1.18.20", "--memory=4096", "--wait=true", "--alsologtostderr", "-v=5"}, StartArgs()...)
rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
if err != nil {
t.Errorf("failed to start minikube with args: %q : %v", rr.Command(), err)
}
})

t.Run("serial", func(t *testing.T) {
tests := []struct {
name string
validator validateFunc
}{
{"ValidateIngressAddonActivation", validateIngressAddonActivation},
{"ValidateIngressDNSAddonActivation", validateIngressDNSAddonActivation},
{"ValidateIngressAddons", validateIngressAddon},
}
for _, tc := range tests {
tc := tc
if ctx.Err() == context.DeadlineExceeded {
t.Fatalf("Unable to run more tests (deadline exceeded)")
}
t.Run(tc.name, func(t *testing.T) {
tc.validator(ctx, t, profile)
})
}
})
}

// validateIngressAddonActivation tests ingress addon activation
func validateIngressAddonActivation(ctx context.Context, t *testing.T, profile string) {
defer PostMortemLogs(t, profile)

if _, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "enable", "ingress", "--alsologtostderr", "-v=5")); err != nil {
t.Errorf("failed to enable ingress addon: %v", err)
}
}

// validateIngressDNSAddonActivation tests ingress-dns addon activation
func validateIngressDNSAddonActivation(ctx context.Context, t *testing.T, profile string) {
defer PostMortemLogs(t, profile)

if _, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "enable", "ingress-dns", "--alsologtostderr", "-v=5")); err != nil {
t.Errorf("failed to enable ingress-dns addon: %v", err)
}
}
85 changes: 85 additions & 0 deletions test/integration/testdata/ingress-dns-example-v1beta1.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
# Copyright 2021 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

---
apiVersion: apps/v1
kind: Deployment
metadata:
name: hello-world-app
namespace: default
spec:
selector:
matchLabels:
app: hello-world-app
template:
metadata:
labels:
app: hello-world-app
spec:
containers:
- name: hello-world-app
image: gcr.io/google-samples/hello-app:1.0
ports:
- containerPort: 8080
---
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: example-ingress
namespace: kube-system
annotations:
# needed for k8s < v1.18: https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#deprecating-the-ingress-class-annotation
kubernetes.io/ingress.class: nginx
spec:
rules:
- host: hello-john.test
http:
paths:
- path: /
pathType: Prefix
backend:
serviceName: hello-world-app
servicePort: 80
- host: hello-jane.test
http:
paths:
- path: /
pathType: Prefix
backend:
serviceName: hello-world-app
servicePort: 80
---
apiVersion: v1
kind: Service
metadata:
name: hello-world-app
namespace: kube-system
spec:
type: ExternalName
externalName: hello-world-app.default.svc.cluster.local
---
apiVersion: v1
kind: Service
metadata:
name: hello-world-app
namespace: default
spec:
ports:
- name: http
port: 80
targetPort: 8080
protocol: TCP
type: NodePort
selector:
app: hello-world-app
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,6 @@ apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: nginx-ingress
annotations:
# use the shared ingress-nginx
kubernetes.io/ingress.class: nginx
labels:
integration-test: ingress
spec:
Expand Down
19 changes: 19 additions & 0 deletions test/integration/testdata/nginx-ingress-v1beta1.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: nginx-ingress
annotations:
# needed for k8s < v1.18: https://kubernetes.io/blog/2020/04/02/improvements-to-the-ingress-api-in-kubernetes-1.18/#deprecating-the-ingress-class-annotation
kubernetes.io/ingress.class: nginx
labels:
integration-test: ingress
spec:
rules:
- host: nginx.example.com
http:
paths:
- path: /
pathType: Prefix
backend:
serviceName: nginx
servicePort: 80

0 comments on commit 6bbf98d

Please sign in to comment.