diff --git a/go.mod b/go.mod index c27c887ed245b..b9220f4959d15 100644 --- a/go.mod +++ b/go.mod @@ -97,6 +97,7 @@ require ( github.com/zclconf/go-cty v1.3.1 go.uber.org/zap v1.10.0 golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 + golang.org/x/net v0.0.0-20200202094626-16171245cfb2 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9 golang.org/x/tools v0.0.0-20191203134012-c197fd4bf371 diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index ba144e248effc..33d34d37622dd 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -2668,6 +2668,22 @@ spec: binary Felix uses Default: Auto (other options: Legacy, NFT)' type: string + ipv4AutoDetectionMethod: + description: 'IPv4AutoDetectionMethod configures how Calico + chooses the IP address used to route between nodes. This + should be set when the host has multiple interfaces and + it is important to select the interface used. Options: "first-found" + (default), "can-reach=DESTINATION", "interface=INTERFACE-REGEX", + or "skip-interface=INTERFACE-REGEX"' + type: string + ipv6AutoDetectionMethod: + description: 'IPv6AutoDetectionMethod configures how Calico + chooses the IP address used to route between nodes. This + should be set when the host has multiple interfaces and + it is important to select the interface used. Options: "first-found" + (default), "can-reach=DESTINATION", "interface=INTERFACE-REGEX", + or "skip-interface=INTERFACE-REGEX"' + type: string logSeverityScreen: description: 'LogSeverityScreen lets us set the desired log level. (Default: info)' diff --git a/pkg/apis/kops/networking.go b/pkg/apis/kops/networking.go index 644feaba8fe0e..52e506d47b7ff 100644 --- a/pkg/apis/kops/networking.go +++ b/pkg/apis/kops/networking.go @@ -118,6 +118,18 @@ type CalicoNetworkingSpec struct { PrometheusProcessMetricsEnabled bool `json:"prometheusProcessMetricsEnabled,omitempty"` // MajorVersion is the version of Calico to use MajorVersion string `json:"majorVersion,omitempty"` + // IPv4AutoDetectionMethod configures how Calico chooses the IP address used to route + // between nodes. This should be set when the host has multiple interfaces + // and it is important to select the interface used. + // Options: "first-found" (default), "can-reach=DESTINATION", + // "interface=INTERFACE-REGEX", or "skip-interface=INTERFACE-REGEX" + IPv4AutoDetectionMethod string `json:"ipv4AutoDetectionMethod,omitempty"` + // IPv6AutoDetectionMethod configures how Calico chooses the IP address used to route + // between nodes. This should be set when the host has multiple interfaces + // and it is important to select the interface used. + // Options: "first-found" (default), "can-reach=DESTINATION", + // "interface=INTERFACE-REGEX", or "skip-interface=INTERFACE-REGEX" + IPv6AutoDetectionMethod string `json:"ipv6AutoDetectionMethod,omitempty"` // IptablesBackend controls which variant of iptables binary Felix uses // Default: Auto (other options: Legacy, NFT) IptablesBackend string `json:"iptablesBackend,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/networking.go b/pkg/apis/kops/v1alpha2/networking.go index 84bac3c0a283c..fe9761f3363c8 100644 --- a/pkg/apis/kops/v1alpha2/networking.go +++ b/pkg/apis/kops/v1alpha2/networking.go @@ -123,6 +123,18 @@ type CalicoNetworkingSpec struct { IptablesBackend string `json:"iptablesBackend,omitempty"` // IPIPMode is mode for CALICO_IPV4POOL_IPIP IPIPMode string `json:"ipipMode,omitempty"` + // IPv4AutoDetectionMethod configures how Calico chooses the IP address used to route + // between nodes. This should be set when the host has multiple interfaces + // and it is important to select the interface used. + // Options: "first-found" (default), "can-reach=DESTINATION", + // "interface=INTERFACE-REGEX", or "skip-interface=INTERFACE-REGEX" + IPv4AutoDetectionMethod string `json:"ipv4AutoDetectionMethod,omitempty"` + // IPv6AutoDetectionMethod configures how Calico chooses the IP address used to route + // between nodes. This should be set when the host has multiple interfaces + // and it is important to select the interface used. + // Options: "first-found" (default), "can-reach=DESTINATION", + // "interface=INTERFACE-REGEX", or "skip-interface=INTERFACE-REGEX" + IPv6AutoDetectionMethod string `json:"ipv6AutoDetectionMethod,omitempty"` // TyphaPrometheusMetricsEnabled enables Prometheus metrics collection from Typha // (default: false) TyphaPrometheusMetricsEnabled bool `json:"typhaPrometheusMetricsEnabled,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index d3f72e7f94d0a..1ede7f5c07fce 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -1292,6 +1292,8 @@ func autoConvert_v1alpha2_CalicoNetworkingSpec_To_kops_CalicoNetworkingSpec(in * out.MajorVersion = in.MajorVersion out.IptablesBackend = in.IptablesBackend out.IPIPMode = in.IPIPMode + out.IPv4AutoDetectionMethod = in.IPv4AutoDetectionMethod + out.IPv6AutoDetectionMethod = in.IPv6AutoDetectionMethod out.TyphaPrometheusMetricsEnabled = in.TyphaPrometheusMetricsEnabled out.TyphaPrometheusMetricsPort = in.TyphaPrometheusMetricsPort out.TyphaReplicas = in.TyphaReplicas @@ -1313,6 +1315,8 @@ func autoConvert_kops_CalicoNetworkingSpec_To_v1alpha2_CalicoNetworkingSpec(in * out.PrometheusGoMetricsEnabled = in.PrometheusGoMetricsEnabled out.PrometheusProcessMetricsEnabled = in.PrometheusProcessMetricsEnabled out.MajorVersion = in.MajorVersion + out.IPv4AutoDetectionMethod = in.IPv4AutoDetectionMethod + out.IPv6AutoDetectionMethod = in.IPv6AutoDetectionMethod out.IptablesBackend = in.IptablesBackend out.IPIPMode = in.IPIPMode out.TyphaPrometheusMetricsEnabled = in.TyphaPrometheusMetricsEnabled diff --git a/pkg/apis/kops/validation/BUILD.bazel b/pkg/apis/kops/validation/BUILD.bazel index 27a02ca2a4e53..dc019862e894e 100644 --- a/pkg/apis/kops/validation/BUILD.bazel +++ b/pkg/apis/kops/validation/BUILD.bazel @@ -24,6 +24,8 @@ go_library( "//upup/pkg/fi/cloudup/awsup:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws/arn:go_default_library", "//vendor/github.com/blang/semver:go_default_library", + "//vendor/golang.org/x/net/ipv4:go_default_library", + "//vendor/golang.org/x/net/ipv6:go_default_library", "//vendor/k8s.io/apimachinery/pkg/api/validation:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//vendor/k8s.io/apimachinery/pkg/util/net:go_default_library", diff --git a/pkg/apis/kops/validation/validation.go b/pkg/apis/kops/validation/validation.go index cb9cf371b6aa3..da087638f27e2 100644 --- a/pkg/apis/kops/validation/validation.go +++ b/pkg/apis/kops/validation/validation.go @@ -17,17 +17,22 @@ limitations under the License. package validation import ( + "errors" "fmt" "net" + "regexp" "strings" "github.com/blang/semver" + "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/kops/upup/pkg/fi" "k8s.io/apimachinery/pkg/api/validation" utilnet "k8s.io/apimachinery/pkg/util/net" "k8s.io/apimachinery/pkg/util/sets" + utilvalidation "k8s.io/apimachinery/pkg/util/validation" "k8s.io/apimachinery/pkg/util/validation/field" "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/pkg/model/components" @@ -699,9 +704,76 @@ func validateNetworkingCalico(v *kops.CalicoNetworkingSpec, e *kops.EtcdClusterS allErrs = append(allErrs, IsValidValue(fldPath.Child("iptablesBackend"), &v.IptablesBackend, valid)...) } + if v.IPv4AutoDetectionMethod != "" { + allErrs = append(allErrs, validateCalicoAutoDetectionMethod(fldPath.Child("ipv4AutoDetectionMethod"), v.IPv4AutoDetectionMethod, ipv4.Version)...) + } + + if v.IPv6AutoDetectionMethod != "" { + allErrs = append(allErrs, validateCalicoAutoDetectionMethod(fldPath.Child("ipv6AutoDetectionMethod"), v.IPv6AutoDetectionMethod, ipv6.Version)...) + } + return allErrs } +func validateCalicoAutoDetectionMethod(fldPath *field.Path, runtime string, version int) field.ErrorList { + validationError := field.ErrorList{} + + // validation code is based on the checks in calico/node startup code + // valid formats are "first-found", "can-reach=DEST", or + // "(skip-)interface=" + // + // We won't do deep validation of the values in this check, since they can + // be actual interface names or regexes + method := strings.Split(runtime, "=") + if len(method) == 0 { + return field.ErrorList{field.Invalid(fldPath, runtime, "missing autodetection method")} + } + if len(method) > 2 { + return field.ErrorList{field.Invalid(fldPath, runtime, "malformed autodetection method")} + } + + // 'method' should contain something like "[interface eth0,en.*]" or "[first-found]" + switch method[0] { + case "first-found": + return nil + case "can-reach": + destStr := method[1] + if version == ipv4.Version { + return utilvalidation.IsValidIPv4Address(fldPath, destStr) + } else if version == ipv6.Version { + return utilvalidation.IsValidIPv6Address(fldPath, destStr) + } + + return field.ErrorList{field.InternalError(fldPath, errors.New("IP version is incorrect"))} + case "interface": + ifRegexes := regexp.MustCompile(`\s*,\s*`).Split(method[1], -1) + if len(ifRegexes) == 0 || ifRegexes[0] == "" { + validationError = append(validationError, field.Invalid(fldPath, runtime, "'interface=' must be followed by a comma separated list of interface regular expressions")) + } + for _, r := range ifRegexes { + _, e := regexp.Compile(r) + if e != nil { + validationError = append(validationError, field.Invalid(fldPath, runtime, fmt.Sprintf("regexp %s does not compile: %s", r, e.Error()))) + } + } + return validationError + case "skip-interface": + ifRegexes := regexp.MustCompile(`\s*,\s*`).Split(method[1], -1) + if len(ifRegexes) == 0 || ifRegexes[0] == "" { + validationError = append(validationError, field.Invalid(fldPath, runtime, "'skip-interface=' must be followed by a comma separated list of interface regular expressions")) + } + for _, r := range ifRegexes { + _, e := regexp.Compile(r) + if e != nil { + validationError = append(validationError, field.Invalid(fldPath, runtime, fmt.Sprintf("regexp %s does not compile: %s", r, e.Error()))) + } + } + return validationError + default: + return field.ErrorList{field.Invalid(fldPath, runtime, "unsupported autodetection method")} + } +} + func validateContainerRuntime(runtime *string, fldPath *field.Path) field.ErrorList { valid := []string{"containerd", "docker"} diff --git a/pkg/apis/kops/validation/validation_test.go b/pkg/apis/kops/validation/validation_test.go index e2448f10f4fc6..094616612c982 100644 --- a/pkg/apis/kops/validation/validation_test.go +++ b/pkg/apis/kops/validation/validation_test.go @@ -408,6 +408,108 @@ func Test_Validate_Calico(t *testing.T) { }, ExpectedErrors: []string{"Forbidden::calico.majorVersion"}, }, + { + Input: caliInput{ + Calico: &kops.CalicoNetworkingSpec{ + IPv4AutoDetectionMethod: "first-found", + }, + Etcd: &kops.EtcdClusterSpec{}, + }, + }, + { + Input: caliInput{ + Calico: &kops.CalicoNetworkingSpec{ + IPv6AutoDetectionMethod: "first-found", + }, + Etcd: &kops.EtcdClusterSpec{}, + }, + }, + { + Input: caliInput{ + Calico: &kops.CalicoNetworkingSpec{ + IPv4AutoDetectionMethod: "can-reach=8.8.8.8", + }, + Etcd: &kops.EtcdClusterSpec{}, + }, + }, + { + Input: caliInput{ + Calico: &kops.CalicoNetworkingSpec{ + IPv6AutoDetectionMethod: "can-reach=2001:4860:4860::8888", + }, + Etcd: &kops.EtcdClusterSpec{}, + }, + }, + { + Input: caliInput{ + Calico: &kops.CalicoNetworkingSpec{ + IPv4AutoDetectionMethod: "bogus", + }, + Etcd: &kops.EtcdClusterSpec{}, + }, + ExpectedErrors: []string{"Invalid value::calico.ipv4AutoDetectionMethod"}, + }, + { + Input: caliInput{ + Calico: &kops.CalicoNetworkingSpec{ + IPv6AutoDetectionMethod: "bogus", + }, + Etcd: &kops.EtcdClusterSpec{}, + }, + ExpectedErrors: []string{"Invalid value::calico.ipv6AutoDetectionMethod"}, + }, + { + Input: caliInput{ + Calico: &kops.CalicoNetworkingSpec{ + IPv6AutoDetectionMethod: "interface=", + }, + Etcd: &kops.EtcdClusterSpec{}, + }, + ExpectedErrors: []string{"Invalid value::calico.ipv6AutoDetectionMethod"}, + }, + { + Input: caliInput{ + Calico: &kops.CalicoNetworkingSpec{ + IPv4AutoDetectionMethod: "interface=en.*,eth0", + }, + Etcd: &kops.EtcdClusterSpec{}, + }, + }, + { + Input: caliInput{ + Calico: &kops.CalicoNetworkingSpec{ + IPv6AutoDetectionMethod: "skip-interface=en.*,eth0", + }, + Etcd: &kops.EtcdClusterSpec{}, + }, + }, + { + Input: caliInput{ + Calico: &kops.CalicoNetworkingSpec{ + IPv4AutoDetectionMethod: "interface=(,en1", + }, + Etcd: &kops.EtcdClusterSpec{}, + }, + ExpectedErrors: []string{"Invalid value::calico.ipv4AutoDetectionMethod"}, + }, + { + Input: caliInput{ + Calico: &kops.CalicoNetworkingSpec{ + IPv4AutoDetectionMethod: "interface=foo=bar", + }, + Etcd: &kops.EtcdClusterSpec{}, + }, + ExpectedErrors: []string{"Invalid value::calico.ipv4AutoDetectionMethod"}, + }, + { + Input: caliInput{ + Calico: &kops.CalicoNetworkingSpec{ + IPv4AutoDetectionMethod: "=en0,eth.*", + }, + Etcd: &kops.EtcdClusterSpec{}, + }, + ExpectedErrors: []string{"Invalid value::calico.ipv4AutoDetectionMethod"}, + }, } for _, g := range grid { errs := validateNetworkingCalico(g.Input.Calico, g.Input.Etcd, field.NewPath("calico")) diff --git a/upup/models/bindata.go b/upup/models/bindata.go index 47e754af7a49d..3122110b6e7fe 100644 --- a/upup/models/bindata.go +++ b/upup/models/bindata.go @@ -7600,6 +7600,10 @@ spec: # Auto-detect the BGP IP address. - name: IP value: "autodetect" + - name: IP_AUTODETECTION_METHOD + value: "{{- or .Networking.Calico.IPv4AutoDetectionMethod "first-found" }}" + - name: IP6_AUTODETECTION_METHOD + value: "{{- or .Networking.Calico.IPv6AutoDetectionMethod "first-found" }}" # Enable IPIP - name: CALICO_IPV4POOL_IPIP value: "{{- if and (eq .CloudProvider "aws") (.Networking.Calico.CrossSubnet) -}}CrossSubnet{{- else -}} {{- or .Networking.Calico.IPIPMode "Always" -}} {{- end -}}" @@ -8715,6 +8719,10 @@ spec: # Auto-detect the BGP IP address. - name: IP value: "autodetect" + - name: IP_AUTODETECTION_METHOD + value: "{{- or .Networking.Calico.IPv4AutoDetectionMethod "first-found" }}" + - name: IP6_AUTODETECTION_METHOD + value: "{{- or .Networking.Calico.IPv6AutoDetectionMethod "first-found" }}" # Enable IPIP - name: CALICO_IPV4POOL_IPIP value: "{{- if and (eq .CloudProvider "aws") (.Networking.Calico.CrossSubnet) -}}CrossSubnet{{- else -}} {{- or .Networking.Calico.IPIPMode "Always" -}} {{- end -}}" diff --git a/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.12.yaml.template b/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.12.yaml.template index 1440d7957c8ee..6a7cfbf738d1a 100644 --- a/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.12.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.12.yaml.template @@ -789,6 +789,10 @@ spec: # Auto-detect the BGP IP address. - name: IP value: "autodetect" + - name: IP_AUTODETECTION_METHOD + value: "{{- or .Networking.Calico.IPv4AutoDetectionMethod "first-found" }}" + - name: IP6_AUTODETECTION_METHOD + value: "{{- or .Networking.Calico.IPv6AutoDetectionMethod "first-found" }}" # Enable IPIP - name: CALICO_IPV4POOL_IPIP value: "{{- if and (eq .CloudProvider "aws") (.Networking.Calico.CrossSubnet) -}}CrossSubnet{{- else -}} {{- or .Networking.Calico.IPIPMode "Always" -}} {{- end -}}" diff --git a/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template b/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template index 722bf53cb8bb0..aae5bc6625331 100644 --- a/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.projectcalico.org/k8s-1.16.yaml.template @@ -800,6 +800,10 @@ spec: # Auto-detect the BGP IP address. - name: IP value: "autodetect" + - name: IP_AUTODETECTION_METHOD + value: "{{- or .Networking.Calico.IPv4AutoDetectionMethod "first-found" }}" + - name: IP6_AUTODETECTION_METHOD + value: "{{- or .Networking.Calico.IPv6AutoDetectionMethod "first-found" }}" # Enable IPIP - name: CALICO_IPV4POOL_IPIP value: "{{- if and (eq .CloudProvider "aws") (.Networking.Calico.CrossSubnet) -}}CrossSubnet{{- else -}} {{- or .Networking.Calico.IPIPMode "Always" -}} {{- end -}}" diff --git a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go index 20bce40d7513b..446c8b7542b4b 100644 --- a/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go +++ b/upup/pkg/fi/cloudup/bootstrapchannelbuilder.go @@ -725,8 +725,8 @@ func (b *BootstrapChannelBuilder) buildAddons() *channelsapi.Addons { versions := map[string]string{ "k8s-1.7": "2.6.12-kops.1", "k8s-1.7-v3": "3.8.0-kops.2", - "k8s-1.12": "3.9.5-kops.3", - "k8s-1.16": "3.13.3-kops.2", + "k8s-1.12": "3.9.5-kops.4", + "k8s-1.16": "3.13.3-kops.3", } {