Skip to content

Commit

Permalink
Implement flag --experimental-set-member-localaddr
Browse files Browse the repository at this point in the history
Which sets the LocalAddr to an IP address from --initial-advertise-peer-urls.

Also adds e2e test that requires this flag to succeed.

Co-authored-by: HighPon <s.shiraki.business@gmail.com>
Signed-off-by: Edwin Xie <edwin.xie@broadcom.com>
  • Loading branch information
flawedmatrix and highpon committed May 24, 2024
1 parent 7a3ca53 commit 4c77726
Show file tree
Hide file tree
Showing 10 changed files with 435 additions and 1 deletion.
1 change: 1 addition & 0 deletions CHANGELOG/CHANGELOG-3.6.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0...v3.6.0).
- Decreased [`--snapshot-count` default value from 100,000 to 10,000](https://github.com/etcd-io/etcd/pull/15408)
- Add [`etcd --tls-min-version --tls-max-version`](https://github.com/etcd-io/etcd/pull/15156) to enable support for TLS 1.3.
- Add [quota to endpoint status response](https://github.com/etcd-io/etcd/pull/17877)
- Add ['etcd --experimental-set-member-localaddr'](https://github.com/etcd-io/etcd/pull/17661) to enable using the first specified and non-loopback local address from initial-advertise-peer-urls as the local address when communicating with a peer.

### etcd grpc-proxy

Expand Down
3 changes: 3 additions & 0 deletions client/pkg/transport/listener.go
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,9 @@ type TLSInfo struct {
// EmptyCN indicates that the cert must have empty CN.
// If true, ClientConfig() will return an error for a cert with non empty CN.
EmptyCN bool

// LocalAddr is the local IP address to use when communicating with a peer.
LocalAddr string
}

func (info TLSInfo) String() string {
Expand Down
11 changes: 10 additions & 1 deletion client/pkg/transport/transport.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,19 @@ func NewTransport(info TLSInfo, dialtimeoutd time.Duration) (*http.Transport, er
return nil, err
}

var ipAddr net.Addr
if info.LocalAddr != "" {
ipAddr, err = net.ResolveTCPAddr("tcp", info.LocalAddr+":0")
if err != nil {
return nil, err
}
}

t := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: dialtimeoutd,
Timeout: dialtimeoutd,
LocalAddr: ipAddr,
// value taken from http.DefaultTransport
KeepAlive: 30 * time.Second,
}).DialContext,
Expand Down
3 changes: 3 additions & 0 deletions server/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,9 @@ type ServerConfig struct {

// V2Deprecation defines a phase of v2store deprecation process.
V2Deprecation V2DeprecationEnum `json:"v2-deprecation"`

// ExperimentalLocalAddress is the local IP address to use when communicating with a peer.
ExperimentalLocalAddress string `json:"experimental-local-address"`
}

// VerifyBootstrap sanity-checks the initial config for bootstrap case
Expand Down
43 changes: 43 additions & 0 deletions server/embed/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"math"
"net"
"net/http"
"net/netip"
"net/url"
"os"
"path/filepath"
Expand Down Expand Up @@ -225,6 +226,12 @@ type Config struct {
ClientAutoTLS bool
PeerTLSInfo transport.TLSInfo
PeerAutoTLS bool

// ExperimentalSetMemberLocalAddr enables using the first specified and
// non-loopback local address from initial-advertise-peer-urls as the local
// address when communicating with a peer.
ExperimentalSetMemberLocalAddr bool `json:"experimental-set-member-localaddr"`

// SelfSignedCertValidity specifies the validity period of the client and peer certificates
// that are automatically generated by etcd when you specify ClientAutoTLS and PeerAutoTLS,
// the unit is year, and the default is 1
Expand Down Expand Up @@ -621,6 +628,8 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) {
"initial-advertise-peer-urls",
"List of this member's peer URLs to advertise to the rest of the cluster.",
)
fs.BoolVar(&cfg.ExperimentalSetMemberLocalAddr, "experimental-set-member-localaddr", false, "Enable to have etcd use the first specified and non-loopback host from initial-advertise-peer-urls as the local address when communicating with a peer.")

fs.Var(
flags.NewUniqueURLsWithExceptions(DefaultAdvertiseClientURLs, ""),
"advertise-client-urls",
Expand Down Expand Up @@ -1148,6 +1157,40 @@ func (cfg *Config) InitialClusterFromName(name string) (ret string) {
return ret[1:]
}

// InferLocalAddr tries to determine the LocalAddr used when communicating with
// an etcd peer. If SetMemberLocalAddr is true, then it will try to get the host
// from AdvertisePeerUrls by searching for the first URL with a specified
// non-loopback address. Otherwise, it defaults to empty string and the
// LocalAddr used will be the default for the Golang HTTP client.
func (cfg *Config) InferLocalAddr() string {
if !cfg.ExperimentalSetMemberLocalAddr {
return ""
}

lg := cfg.GetLogger()
lg.Info(
"searching for a suitable member local address in AdvertisePeerURLs",
zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerURLs()),
)
for _, peerURL := range cfg.AdvertisePeerUrls {
if addr, err := netip.ParseAddr(peerURL.Hostname()); err == nil {
if addr.IsLoopback() || addr.IsUnspecified() {
continue
}
lg.Info(
"setting member local address",
zap.String("LocalAddr", addr.String()),
)
return addr.String()
}
}
lg.Warn(
"unable to set a member local address due to lack of suitable local addresses",
zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerURLs()),
)
return ""
}

func (cfg *Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateFlagNew }
func (cfg *Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) }

Expand Down
132 changes: 132 additions & 0 deletions server/embed/config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,138 @@ func TestUpdateDefaultClusterFromNameOverwrite(t *testing.T) {
}
}

func TestInferLocalAddr(t *testing.T) {
tests := []struct {
name string
advertisePeerURLs []string
setMemberLocalAddr bool
expectedLocalAddr string
}{
{
"defaults, ExperimentalSetMemberLocalAddr=false ",
[]string{DefaultInitialAdvertisePeerURLs},
false,
"",
},
{
"IPv4 address, ExperimentalSetMemberLocalAddr=false ",
[]string{"https://192.168.100.110:2380"},
false,
"",
},
{
"defaults, ExperimentalSetMemberLocalAddr=true",
[]string{DefaultInitialAdvertisePeerURLs},
true,
"",
},
{
"IPv4 unspecified address, ExperimentalSetMemberLocalAddr=true",
[]string{"https://0.0.0.0:2380"},
true,
"",
},
{
"IPv6 unspecified address, ExperimentalSetMemberLocalAddr=true",
[]string{"https://[::]:2380"},
true,
"",
},
{
"IPv4 loopback address, ExperimentalSetMemberLocalAddr=true",
[]string{"https://127.0.0.1:2380"},
true,
"",
},
{
"IPv6 loopback address, ExperimentalSetMemberLocalAddr=true",
[]string{"https://[::1]:2380"},
true,
"",
},
{
"IPv4 address, ExperimentalSetMemberLocalAddr=true",
[]string{"https://192.168.100.110:2380"},
true,
"192.168.100.110",
},
{
"Hostname only, ExperimentalSetMemberLocalAddr=true",
[]string{"https://123-host-3.corp.internal:2380"},
true,
"",
},
{
"Hostname and IPv4 address, ExperimentalSetMemberLocalAddr=true",
[]string{"https://123-host-3.corp.internal:2380", "https://192.168.100.110:2380"},
true,
"192.168.100.110",
},
{
"IPv4 address and Hostname, ExperimentalSetMemberLocalAddr=true",
[]string{"https://192.168.100.110:2380", "https://123-host-3.corp.internal:2380"},
true,
"192.168.100.110",
},
{
"IPv4 and IPv6 addresses, ExperimentalSetMemberLocalAddr=true",
[]string{"https://192.168.100.110:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"},
true,
"192.168.100.110",
},
{
"IPv6 and IPv4 addresses, ExperimentalSetMemberLocalAddr=true",
// IPv4 addresses will always sort before IPv6 ones anyway
[]string{"https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://192.168.100.110:2380"},
true,
"192.168.100.110",
},
{
"Hostname, IPv4 and IPv6 addresses, ExperimentalSetMemberLocalAddr=true",
[]string{"https://123-host-3.corp.internal:2380", "https://192.168.100.110:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"},
true,
"192.168.100.110",
},
{
"Hostname, IPv6 and IPv4 addresses, ExperimentalSetMemberLocalAddr=true",
// IPv4 addresses will always sort before IPv6 ones anyway
[]string{"https://123-host-3.corp.internal:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://192.168.100.110:2380"},
true,
"192.168.100.110",
},
{
"IPv6 address, ExperimentalSetMemberLocalAddr=true",
[]string{"https://[2001:db8:85a3::8a2e:370:7334]:2380"},
true,
"2001:db8:85a3::8a2e:370:7334",
},
{
"Hostname and IPv6 address, ExperimentalSetMemberLocalAddr=true",
[]string{"https://123-host-3.corp.internal:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"},
true,
"2001:db8:85a3::8a2e:370:7334",
},
{
"IPv6 address and Hostname, ExperimentalSetMemberLocalAddr=true",
[]string{"https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://123-host-3.corp.internal:2380"},
true,
"2001:db8:85a3::8a2e:370:7334",
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := NewConfig()
cfg.AdvertisePeerUrls = types.MustNewURLs(tt.advertisePeerURLs)
cfg.ExperimentalSetMemberLocalAddr = tt.setMemberLocalAddr

require.NoError(t, cfg.Validate())
require.Equal(t, tt.expectedLocalAddr, cfg.InferLocalAddr())
})
}

}

func (s *securityConfig) equals(t *transport.TLSInfo) bool {
return s.CertFile == t.CertFile &&
s.CertAuth == t.ClientCertAuth &&
Expand Down
5 changes: 5 additions & 0 deletions server/embed/etcd.go
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
ExperimentalBootstrapDefragThresholdMegabytes: cfg.ExperimentalBootstrapDefragThresholdMegabytes,
ExperimentalMaxLearners: cfg.ExperimentalMaxLearners,
V2Deprecation: cfg.V2DeprecationEffective(),
ExperimentalLocalAddress: cfg.InferLocalAddr(),
}

if srvcfg.ExperimentalEnableDistributedTracing {
Expand All @@ -245,6 +246,8 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
)
}

srvcfg.PeerTLSInfo.LocalAddr = srvcfg.ExperimentalLocalAddress

print(e.cfg.logger, *cfg, srvcfg, memberInitialized)

if e.Server, err = etcdserver.NewServer(srvcfg); err != nil {
Expand Down Expand Up @@ -336,6 +339,8 @@ func print(lg *zap.Logger, ec Config, sc config.ServerConfig, memberInitialized
zap.Strings("advertise-client-urls", ec.getAdvertiseClientURLs()),
zap.Strings("listen-client-urls", ec.getListenClientURLs()),
zap.Strings("listen-metrics-urls", ec.getMetricsURLs()),
zap.Bool("experimental-set-member-localaddr", ec.ExperimentalSetMemberLocalAddr),
zap.String("experimental-local-address", sc.ExperimentalLocalAddress),
zap.Strings("cors", cors),
zap.Strings("host-whitelist", hss),
zap.String("initial-cluster", sc.InitialPeerURLsMap.String()),
Expand Down
2 changes: 2 additions & 0 deletions server/etcdmain/help.go
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,8 @@ Member:
Clustering:
--initial-advertise-peer-urls 'http://localhost:2380'
List of this member's peer URLs to advertise to the rest of the cluster.
--experimental-set-member-localaddr 'false'
Enable using the first specified and non-loopback local address from initial-advertise-peer-urls as the local address when communicating with a peer.
--initial-cluster 'default=http://localhost:2380'
Initial cluster configuration for bootstrapping.
--initial-cluster-state 'new'
Expand Down
Loading

0 comments on commit 4c77726

Please sign in to comment.