Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

bug 1804913: make ipv6 support a little better #173

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
185 changes: 185 additions & 0 deletions pkg/dnshelpers/util.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
package dnshelpers

import (
"fmt"
"net"
"strings"

configv1 "github.com/openshift/api/config/v1"

"k8s.io/apimachinery/pkg/util/sets"

corev1 "k8s.io/api/core/v1"

utilerrors "k8s.io/apimachinery/pkg/util/errors"

"k8s.io/klog"
)

// GetEscapedPreferredInternalIPAddressForNodeName returns the first internal ip address of the correct family with escaping
// for ipv6.
func GetEscapedPreferredInternalIPAddressForNodeName(network *configv1.Network, node *corev1.Node) (string, error) {
address, family, err := GetPreferredInternalIPAddressForNodeName(network, node)
if err != nil {
return "", err
}
if family == "tcp6" {
return "[" + address + "]", nil
} else {
return address, nil
}
}

// GetPreferredInternalIPAddressForNodeName returns the first internal ip address of the correct family and the family
func GetPreferredInternalIPAddressForNodeName(network *configv1.Network, node *corev1.Node) (string, string, error) {
ipFamily, err := GetPreferredIPFamily(network)
if err != nil {
return "", "", err
}

for _, currAddress := range node.Status.Addresses {
if currAddress.Type == corev1.NodeInternalIP {
switch ipFamily {
case "tcp4":
isIPv4, err := IsIPv4(currAddress.Address)
if err != nil {
return "", "", err
}
if isIPv4 {
return currAddress.Address, ipFamily, nil
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

if an IPv4 address in IPv6 format, isIPv4 will be true, but you will need the brackets.
net.IP's String() will return a canonical IP address string that will match the ipFamily type.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

node.Status.Addresses should never have IPv4 addresses in IPv6 format (other things will almost definitely break if it does)

}
case "tcp6":
isIPv4, err := IsIPv4(currAddress.Address)
if err != nil {
return "", "", err
}
if !isIPv4 {
return currAddress.Address, ipFamily, nil
}
default:
return "", "", fmt.Errorf("unexpected ip family: %q", ipFamily)
}
}
}

return "", "", fmt.Errorf("no matches found for ip family %q for node %q", ipFamily, node.Name)
}

func GetPreferredIPFamily(network *configv1.Network) (string, error) {
if len(network.Status.ServiceNetwork) == 0 || len(network.Status.ServiceNetwork[0]) == 0 {
return "", fmt.Errorf("networks.%s/cluster: status.serviceNetwork not found", configv1.GroupName)
}

serviceCIDR := network.Status.ServiceNetwork[0]
if len(serviceCIDR) == 0 {
return "", fmt.Errorf("networks.%s/cluster: status.serviceNetwork[0] is empty", configv1.GroupName)
}
ip, _, err := net.ParseCIDR(serviceCIDR)

switch {
case err != nil:
return "", err
case ip.To4() == nil:
return "tcp6", nil
default:
return "tcp4", nil
}
}

func IsIPv4(ipString string) (bool, error) {
ip := net.ParseIP(ipString)

switch {
case ip == nil:
return false, fmt.Errorf("not an IP")
case ip.To4() == nil:
return false, nil
default:
return true, nil
}
}

func GetInternalIPAddressesForNodeName(node *corev1.Node) ([]string, error) {
addresses := []string{}
for _, currAddress := range node.Status.Addresses {
if currAddress.Type == corev1.NodeInternalIP {
addresses = append(addresses, currAddress.Address)
}
}
if len(addresses) == 0 {
return nil, fmt.Errorf("node/%s missing %s", node.Name, corev1.NodeInternalIP)
}

return addresses, nil
}

func ReverseLookupFirstHit(discoveryDomain string, ips ...string) (string, error) {
errs := []error{}
for _, ip := range ips {
ret, err := reverseLookupForOneIP(discoveryDomain, ip)
if err == nil {
return ret, nil
}
errs = append(errs, err)
}

if len(errs) == 0 {
return "", fmt.Errorf("something weird happened for %q, %#v", discoveryDomain, ips)
}
return "", utilerrors.NewAggregate(errs)
}

func ReverseLookupAllHits(discoveryDomain string, ips ...string) ([]string, error) {
ret := []string{}
found := sets.NewString()
errs := []error{}
for _, ip := range ips {
curr, err := reverseLookupForOneIP(discoveryDomain, ip)
if err != nil {
errs = append(errs, err)
} else if !found.Has(curr) {
ret = append(ret, curr)
found.Insert(curr)
}
}

switch {
case len(ret) > 0:
//ignore errors
return ret, nil
case len(errs) == 0:
// we got no result and no error
return nil, fmt.Errorf("something weird happened for %q, %#v", discoveryDomain, ips)
default:
// we got errors
return nil, utilerrors.NewAggregate(errs)
}
}
func reverseLookupForOneIP(discoveryDomain, ipAddress string) (string, error) {
service := "etcd-server-ssl"
proto := "tcp"

_, srvs, err := net.LookupSRV(service, proto, discoveryDomain)
if err != nil {
return "", err
}
selfTarget := ""
for _, srv := range srvs {
klog.V(4).Infof("checking against %s", srv.Target)
addrs, err := net.LookupHost(srv.Target)
if err != nil {
return "", fmt.Errorf("could not resolve member %q", srv.Target)
}

for _, addr := range addrs {
if addr == ipAddress {
selfTarget = strings.Trim(srv.Target, ".")
break
}
}
}
if selfTarget == "" {
return "", fmt.Errorf("could not find self")
}
return selfTarget, nil
}
29 changes: 17 additions & 12 deletions pkg/etcdcli/etcdcli.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,15 @@ import (
"fmt"
"time"

configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1"
configv1listers "github.com/openshift/client-go/config/listers/config/v1"
"github.com/openshift/cluster-etcd-operator/pkg/dnshelpers"
"github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient"
"github.com/openshift/library-go/pkg/operator/v1helpers"
"go.etcd.io/etcd/clientv3"
"go.etcd.io/etcd/etcdserver/etcdserverpb"
"go.etcd.io/etcd/pkg/transport"
"google.golang.org/grpc"
corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime/schema"
Expand All @@ -23,17 +25,21 @@ import (
type etcdClientGetter struct {
nodeLister corev1listers.NodeLister
endpointsLister corev1listers.EndpointsLister
networkLister configv1listers.NetworkLister

nodeListerSynced cache.InformerSynced
endpointsListerSynced cache.InformerSynced
networkListerSynced cache.InformerSynced
}

func NewEtcdClient(kubeInformers v1helpers.KubeInformersForNamespaces) EtcdClient {
func NewEtcdClient(kubeInformers v1helpers.KubeInformersForNamespaces, networkInformer configv1informers.NetworkInformer) EtcdClient {
return &etcdClientGetter{
nodeLister: kubeInformers.InformersFor("").Core().V1().Nodes().Lister(),
endpointsLister: kubeInformers.InformersFor(operatorclient.TargetNamespace).Core().V1().Endpoints().Lister(),
networkLister: networkInformer.Lister(),
nodeListerSynced: kubeInformers.InformersFor("").Core().V1().Nodes().Informer().HasSynced,
endpointsListerSynced: kubeInformers.InformersFor(operatorclient.TargetNamespace).Core().V1().Endpoints().Informer().HasSynced,
networkListerSynced: networkInformer.Informer().HasSynced,
}
}

Expand All @@ -44,11 +50,19 @@ func (g *etcdClientGetter) getEtcdClient() (*clientv3.Client, error) {
if !g.endpointsListerSynced() {
return nil, fmt.Errorf("node lister not synced")
}
if !g.networkListerSynced() {
return nil, fmt.Errorf("network lister not synced")
}

network, err := g.networkLister.Get("cluster")
if err != nil {
return nil, err
}

etcdEndpoints := []string{}
nodes, err := g.nodeLister.List(labels.Set{"node-role.kubernetes.io/master": ""}.AsSelector())
for _, node := range nodes {
internalIP, err := getInternalIPAddressForNodeName(node)
internalIP, err := dnshelpers.GetEscapedPreferredInternalIPAddressForNodeName(network, node)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -100,15 +114,6 @@ func getEtcdClient(endpoints []string) (*clientv3.Client, error) {
return cli, err
}

func getInternalIPAddressForNodeName(node *corev1.Node) (string, error) {
for _, currAddress := range node.Status.Addresses {
if currAddress.Type == corev1.NodeInternalIP {
return currAddress.Address, nil
}
}
return "", fmt.Errorf("node/%s missing %s", node.Name, corev1.NodeInternalIP)
}

func (g *etcdClientGetter) MemberAdd(peerURL string) error {
cli, err := g.getEtcdClient()
if err != nil {
Expand Down
20 changes: 6 additions & 14 deletions pkg/operator/clustermembercontroller/clustermembercontroller.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
operatorv1 "github.com/openshift/api/operator/v1"
configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1"
configv1listers "github.com/openshift/client-go/config/listers/config/v1"
"github.com/openshift/cluster-etcd-operator/pkg/dnshelpers"
"github.com/openshift/cluster-etcd-operator/pkg/etcdcli"
"github.com/openshift/library-go/pkg/operator/events"
"github.com/openshift/library-go/pkg/operator/v1helpers"
Expand Down Expand Up @@ -263,31 +264,22 @@ func (c *ClusterMemberController) getValidPodFQDNToScale(podToAdd *corev1.Pod) (
if podToAdd.Spec.NodeName == "" {
return "", fmt.Errorf("node name empty for %s", podToAdd.Name)
}
nodeInternalIP, err := c.getNodeInternalIP(podToAdd.Spec.NodeName)
nodeInternalIPs, err := c.getNodeInternalIPs(podToAdd.Spec.NodeName)
if err != nil {
return "", err
}
podFQDN, err := ReverseLookupSelf("etcd-server-ssl", "tcp", etcdDiscoveryDomain, nodeInternalIP)
podFQDN, err := dnshelpers.ReverseLookupFirstHit(etcdDiscoveryDomain, nodeInternalIPs...)
if err != nil {
return "", err
}

return podFQDN, nil
}

func (c *ClusterMemberController) getNodeInternalIP(nodeName string) (string, error) {
func (c *ClusterMemberController) getNodeInternalIPs(nodeName string) ([]string, error) {
node, err := c.nodeLister.Get(nodeName)
if err != nil {
return "", err
}
if node.Status.Addresses == nil {
return "", fmt.Errorf("cannot get node IP address, addresses for node %s is nil", nodeName)
}

for _, addr := range node.Status.Addresses {
if addr.Type == corev1.NodeInternalIP {
return addr.Address, nil
}
return nil, err
}
return "", fmt.Errorf("unable to get internal IP address for node %s", nodeName)
return dnshelpers.GetInternalIPAddressesForNodeName(node)
}
35 changes: 0 additions & 35 deletions pkg/operator/clustermembercontroller/util.go

This file was deleted.

Loading