From c2e26d81d472b62723b1a7b50f6e4c91a34407eb Mon Sep 17 00:00:00 2001 From: Edwin Xie Date: Sat, 9 Mar 2024 02:39:54 +0000 Subject: [PATCH] Implement flag --set-member-localaddr Which sets the LocalAddr to an IP address from --initial-advertise-peer-urls. Also adds e2e test that requires this flag to succeed. Signed-off-by: Edwin Xie --- client/pkg/transport/listener.go | 2 +- server/config/config.go | 2 +- server/embed/config.go | 48 +++++++++-- server/embed/config_test.go | 132 +++++++++++++++++++++++++++++++ server/embed/etcd.go | 5 ++ server/etcdmain/help.go | 5 +- tests/e2e/etcd_config_test.go | 102 ++++++++++++++++++++++++ tests/e2e/utils.go | 126 +++++++++++++++++++++++++++++ 8 files changed, 412 insertions(+), 10 deletions(-) diff --git a/client/pkg/transport/listener.go b/client/pkg/transport/listener.go index f27d6c2f0ebc..bbb84af4b58a 100644 --- a/client/pkg/transport/listener.go +++ b/client/pkg/transport/listener.go @@ -194,7 +194,7 @@ type TLSInfo struct { // If true, ClientConfig() will return an error for a cert with non empty CN. EmptyCN bool - // LocalAddr is the local IP address to use when communicating peer. + // LocalAddr is the local IP address to use when communicating with a peer. LocalAddr string } diff --git a/server/config/config.go b/server/config/config.go index c5efe3ec3dd5..7c87fd63ed00 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -205,7 +205,7 @@ type ServerConfig struct { // V2Deprecation defines a phase of v2store deprecation process. V2Deprecation V2DeprecationEnum `json:"v2-deprecation"` - // LocalAddr is the local IP address to use when communicating peer. + // LocalAddr is the local IP address to use when communicating with a peer. LocalAddr string `json:"local-address"` } diff --git a/server/embed/config.go b/server/embed/config.go index 13fc8e407669..8a10932b1ff7 100644 --- a/server/embed/config.go +++ b/server/embed/config.go @@ -22,6 +22,7 @@ import ( "math" "net" "net/http" + "net/netip" "net/url" "os" "path/filepath" @@ -225,6 +226,12 @@ type Config struct { ClientAutoTLS bool PeerTLSInfo transport.TLSInfo PeerAutoTLS bool + + // SetMemberLocalAddr specifies if true, that etcd should use the first + // specified and non-loopback host in AdvertisePeerUrls as the LocalAddr when + // communicating with a peer. + SetMemberLocalAddr bool `json:"set-member-localaddr"` + // SelfSignedCertValidity specifies the validity period of the client and peer certificates // that are automatically generated by etcd when you specify ClientAutoTLS and PeerAutoTLS, // the unit is year, and the default is 1 @@ -335,9 +342,6 @@ type Config struct { // AuthTokenTTL in seconds of the simple token AuthTokenTTL uint `json:"auth-token-ttl"` - // PeerLocalAddr is the local IP address to use when communicating peer. - PeerLocalAddr string `json:"peer-local-addr"` - ExperimentalInitialCorruptCheck bool `json:"experimental-initial-corrupt-check"` ExperimentalCorruptCheckTime time.Duration `json:"experimental-corrupt-check-time"` ExperimentalCompactHashCheckEnabled bool `json:"experimental-compact-hash-check-enabled"` @@ -622,6 +626,8 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) { "initial-advertise-peer-urls", "List of this member's peer URLs to advertise to the rest of the cluster.", ) + fs.BoolVar(&cfg.SetMemberLocalAddr, "set-member-localaddr", false, "Enable to have etcd use the first specified and non-loopback host from initial-advertise-peer-urls as the local address when communicating with a peer.") + fs.Var( flags.NewUniqueURLsWithExceptions(DefaultAdvertiseClientURLs, ""), "advertise-client-urls", @@ -673,8 +679,6 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) { fs.StringVar(&cfg.PeerTLSInfo.ClientKeyFile, "peer-client-key-file", "", "Path to an explicit peer client TLS key file otherwise peer key file will be used when client auth is required.") fs.BoolVar(&cfg.PeerTLSInfo.ClientCertAuth, "peer-client-cert-auth", false, "Enable peer client cert authentication.") fs.StringVar(&cfg.PeerTLSInfo.TrustedCAFile, "peer-trusted-ca-file", "", "Path to the peer server TLS trusted CA file.") - fs.StringVar(&cfg.PeerTLSInfo.LocalAddr, "peer-local-addr", "", "peer-local-addr is the local IP address to use when communicating peer.") - fs.BoolVar(&cfg.PeerAutoTLS, "peer-auto-tls", false, "Peer TLS using generated certificates") fs.UintVar(&cfg.SelfSignedCertValidity, "self-signed-cert-validity", 1, "The validity period of the client and peer certificates, unit is year") fs.StringVar(&cfg.PeerTLSInfo.CRLFile, "peer-crl-file", "", "Path to the peer certificate revocation list file.") @@ -1149,6 +1153,40 @@ func (cfg *Config) InitialClusterFromName(name string) (ret string) { return ret[1:] } +// InferLocalAddr tries to determine the LocalAddr used when communicating with +// an etcd peer. If SetMemberLocalAddr is true, then it will try to get the host +// from AdvertisePeerUrls by searching for the first URL with a specified +// non-loopback address. Otherwise, it defaults to empty string and the +// LocalAddr used will be the default for the Golang HTTP client. +func (cfg *Config) InferLocalAddr() string { + if !cfg.SetMemberLocalAddr { + return "" + } + + lg := cfg.GetLogger() + lg.Info( + "searching for a suitable member local address in AdvertisePeerURLs", + zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerURLs()), + ) + for _, peerURL := range cfg.AdvertisePeerUrls { + if addr, err := netip.ParseAddr(peerURL.Hostname()); err == nil { + if addr.IsLoopback() || addr.IsUnspecified() { + continue + } + lg.Info( + "setting member local address", + zap.String("LocalAddr", addr.String()), + ) + return addr.String() + } + } + lg.Warn( + "unable to set a member local address due to lack of suitable local addresses", + zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerURLs()), + ) + return "" +} + func (cfg *Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateFlagNew } func (cfg *Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) } diff --git a/server/embed/config_test.go b/server/embed/config_test.go index 8073a1085ece..0d5f0c5e7fcc 100644 --- a/server/embed/config_test.go +++ b/server/embed/config_test.go @@ -152,6 +152,138 @@ func TestUpdateDefaultClusterFromNameOverwrite(t *testing.T) { } } +func TestInferLocalAddr(t *testing.T) { + tests := []struct { + name string + advertisePeerURLs []string + setMemberLocalAddr bool + expectedLocalAddr string + }{ + { + "defaults, SetMemberLocalAddr=false ", + []string{DefaultInitialAdvertisePeerURLs}, + false, + "", + }, + { + "IPv4 address, SetMemberLocalAddr=false ", + []string{"https://192.168.100.110:2380"}, + false, + "", + }, + { + "defaults, SetMemberLocalAddr=true", + []string{DefaultInitialAdvertisePeerURLs}, + true, + "", + }, + { + "IPv4 unspecified address, SetMemberLocalAddr=true", + []string{"https://0.0.0.0:2380"}, + true, + "", + }, + { + "IPv6 unspecified address, SetMemberLocalAddr=true", + []string{"https://[::]:2380"}, + true, + "", + }, + { + "IPv4 loopback address, SetMemberLocalAddr=true", + []string{"https://127.0.0.1:2380"}, + true, + "", + }, + { + "IPv6 loopback address, SetMemberLocalAddr=true", + []string{"https://[::1]:2380"}, + true, + "", + }, + { + "IPv4 address, SetMemberLocalAddr=true", + []string{"https://192.168.100.110:2380"}, + true, + "192.168.100.110", + }, + { + "Hostname only, SetMemberLocalAddr=true", + []string{"https://123-host-3.corp.internal:2380"}, + true, + "", + }, + { + "Hostname and IPv4 address, SetMemberLocalAddr=true", + []string{"https://123-host-3.corp.internal:2380", "https://192.168.100.110:2380"}, + true, + "192.168.100.110", + }, + { + "IPv4 address and Hostname, SetMemberLocalAddr=true", + []string{"https://192.168.100.110:2380", "https://123-host-3.corp.internal:2380"}, + true, + "192.168.100.110", + }, + { + "IPv4 and IPv6 addresses, SetMemberLocalAddr=true", + []string{"https://192.168.100.110:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"}, + true, + "192.168.100.110", + }, + { + "IPv6 and IPv4 addresses, SetMemberLocalAddr=true", + // IPv4 addresses will always sort before IPv6 ones anyway + []string{"https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://192.168.100.110:2380"}, + true, + "192.168.100.110", + }, + { + "Hostname, IPv4 and IPv6 addresses, SetMemberLocalAddr=true", + []string{"https://123-host-3.corp.internal:2380", "https://192.168.100.110:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"}, + true, + "192.168.100.110", + }, + { + "Hostname, IPv6 and IPv4 addresses, SetMemberLocalAddr=true", + // IPv4 addresses will always sort before IPv6 ones anyway + []string{"https://123-host-3.corp.internal:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://192.168.100.110:2380"}, + true, + "192.168.100.110", + }, + { + "IPv6 address, SetMemberLocalAddr=true", + []string{"https://[2001:db8:85a3::8a2e:370:7334]:2380"}, + true, + "2001:db8:85a3::8a2e:370:7334", + }, + { + "Hostname and IPv6 address, SetMemberLocalAddr=true", + []string{"https://123-host-3.corp.internal:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"}, + true, + "2001:db8:85a3::8a2e:370:7334", + }, + { + "IPv6 address and Hostname, SetMemberLocalAddr=true", + []string{"https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://123-host-3.corp.internal:2380"}, + true, + "2001:db8:85a3::8a2e:370:7334", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := NewConfig() + cfg.AdvertisePeerUrls = types.MustNewURLs(tt.advertisePeerURLs) + cfg.SetMemberLocalAddr = tt.setMemberLocalAddr + + require.NoError(t, cfg.Validate()) + require.Equal(t, tt.expectedLocalAddr, cfg.InferLocalAddr()) + }) + } + +} + func (s *securityConfig) equals(t *transport.TLSInfo) bool { return s.CertFile == t.CertFile && s.CertAuth == t.ClientCertAuth && diff --git a/server/embed/etcd.go b/server/embed/etcd.go index 1b7b3df0cb90..891fe7b8feb8 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -227,6 +227,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { ExperimentalBootstrapDefragThresholdMegabytes: cfg.ExperimentalBootstrapDefragThresholdMegabytes, ExperimentalMaxLearners: cfg.ExperimentalMaxLearners, V2Deprecation: cfg.V2DeprecationEffective(), + LocalAddr: cfg.InferLocalAddr(), } if srvcfg.ExperimentalEnableDistributedTracing { @@ -245,6 +246,8 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { ) } + srvcfg.PeerTLSInfo.LocalAddr = srvcfg.LocalAddr + print(e.cfg.logger, *cfg, srvcfg, memberInitialized) if e.Server, err = etcdserver.NewServer(srvcfg); err != nil { @@ -336,6 +339,8 @@ func print(lg *zap.Logger, ec Config, sc config.ServerConfig, memberInitialized zap.Strings("advertise-client-urls", ec.getAdvertiseClientURLs()), zap.Strings("listen-client-urls", ec.getListenClientURLs()), zap.Strings("listen-metrics-urls", ec.getMetricsURLs()), + zap.Bool("set-member-localaddr", ec.SetMemberLocalAddr), + zap.String("local-addr", sc.LocalAddr), zap.Strings("cors", cors), zap.Strings("host-whitelist", hss), zap.String("initial-cluster", sc.InitialPeerURLsMap.String()), diff --git a/server/etcdmain/help.go b/server/etcdmain/help.go index bdc4964febd6..a24be2d6123e 100644 --- a/server/etcdmain/help.go +++ b/server/etcdmain/help.go @@ -107,6 +107,8 @@ Member: Clustering: --initial-advertise-peer-urls 'http://localhost:2380' List of this member's peer URLs to advertise to the rest of the cluster. + --set-member-localaddr 'false' + Enable using the first specified and non-loopback local address from initial-advertise-peer-urls as the local address when communicating with a peer. --initial-cluster 'default=http://localhost:2380' Initial cluster configuration for bootstrapping. --initial-cluster-state 'new' @@ -221,9 +223,6 @@ Security: Minimum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3. --tls-max-version '' Maximum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3 (empty will be auto-populated by Go). - --peer-local-addr '' - LocalAddr is the local IP address to use when communicating peer. - Auth: --auth-token 'simple' diff --git a/tests/e2e/etcd_config_test.go b/tests/e2e/etcd_config_test.go index aa8370bd344b..bb8f2c0bcb58 100644 --- a/tests/e2e/etcd_config_test.go +++ b/tests/e2e/etcd_config_test.go @@ -17,6 +17,7 @@ package e2e import ( "context" "fmt" + "net" "os" "strings" "testing" @@ -274,6 +275,107 @@ func TestEtcdPeerNameAuth(t *testing.T) { } } +// TestEtcdPeerLocalAddr checks that the inter peer auth works with when +// the member LocalAddr is set. +func TestEtcdPeerLocalAddr(t *testing.T) { + e2e.SkipInShortMode(t) + + nodeIP, err := getLocalIP() + t.Log("Using node IP", nodeIP) + require.NoError(t, err) + + peers, tmpdirs := make([]string, 3), make([]string, 3) + + for i := range peers { + peerIP := nodeIP + if i == 0 { + peerIP = "127.0.0.1" + } + peers[i] = fmt.Sprintf("e%d=https://%s:%d", i, peerIP, e2e.EtcdProcessBasePort+i) + tmpdirs[i] = t.TempDir() + } + procs := make([]*expect.ExpectProcess, len(peers)) + defer func() { + for i := range procs { + if procs[i] != nil { + procs[i].Stop() + procs[i].Close() + } + os.RemoveAll(tmpdirs[i]) + } + }() + + tempDir := t.TempDir() + caFile, certFiles, keyFiles, err := generateCertsForIPs(tempDir, []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP(nodeIP)}) + require.NoError(t, err) + + defer func() { + os.RemoveAll(tempDir) + }() + + // node 0 (127.0.0.1) does not set localaddr, while nodes 1 and nodes 2 (both use host's IP) do. + // The other two nodes will reject connections from node 0 warning that node 0's certificate is valid only for + // 127.0.0.1, not the host IP, since node 0 will try to connect to the other peers with the host IP + // as the client address. + // Node 0 will not reject connections from the other nodes since they will + // use the host's IP to connect (due to --set-member-localaddr) + for i := range procs { + peerIP := nodeIP + if i == 0 { + peerIP = "127.0.0.1" + } + ic := strings.Join(peers, ",") + commonArgs := []string{ + e2e.BinPath.Etcd, + "--name", fmt.Sprintf("e%d", i), + "--listen-client-urls", "http://0.0.0.0:0", + "--data-dir", tmpdirs[i], + "--advertise-client-urls", "http://0.0.0.0:0", + "--initial-advertise-peer-urls", fmt.Sprintf("https://%s:%d", peerIP, e2e.EtcdProcessBasePort+i), + "--listen-peer-urls", fmt.Sprintf("https://%s:%d,https://%s:%d", peerIP, e2e.EtcdProcessBasePort+i, peerIP, e2e.EtcdProcessBasePort+len(peers)+i), + "--initial-cluster", ic, + } + + var args []string + if i == 0 { + args = []string{ + "--peer-cert-file", certFiles[0], + "--peer-key-file", keyFiles[0], + "--peer-trusted-ca-file", caFile, + "--peer-client-cert-auth", + } + } else { + args = []string{ + "--peer-cert-file", certFiles[1], + "--peer-key-file", keyFiles[1], + "--peer-trusted-ca-file", caFile, + "--peer-client-cert-auth", + "--set-member-localaddr", + } + } + + commonArgs = append(commonArgs, args...) + + p, err := e2e.SpawnCmd(commonArgs, nil) + if err != nil { + t.Fatal(err) + } + procs[i] = p + } + + for i, p := range procs { + var expect []string + if i == 0 { + expect = e2e.EtcdServerReadyLines + } else { + expect = []string{"x509: certificate is valid for 127.0.0.1, not "} + } + if err := e2e.WaitReadyExpectProc(context.TODO(), p, expect); err != nil { + t.Fatal(err) + } + } +} + func TestGrpcproxyAndCommonName(t *testing.T) { e2e.SkipInShortMode(t) diff --git a/tests/e2e/utils.go b/tests/e2e/utils.go index cd7c1c90df92..c8ccf9c19902 100644 --- a/tests/e2e/utils.go +++ b/tests/e2e/utils.go @@ -15,9 +15,18 @@ package e2e import ( + "bytes" "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" "encoding/json" + "encoding/pem" "fmt" + "math/big" + "net" + "os" "strings" "testing" "time" @@ -145,3 +154,120 @@ func patchArgs(args []string, flag, newValue string) error { } return fmt.Errorf("--%s flag not found", flag) } + +func generateCertsForIPs(tempDir string, ips []net.IP) (caFile string, certFiles []string, keyFiles []string, err error) { + ca := &x509.Certificate{ + SerialNumber: big.NewInt(1001), + Subject: pkix.Name{ + Organization: []string{"etcd"}, + OrganizationalUnit: []string{"etcd Security"}, + Locality: []string{"San Francisco"}, + Province: []string{"California"}, + Country: []string{"USA"}, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(0, 0, 1), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return "", nil, nil, err + } + caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, &caKey.PublicKey, caKey) + if err != nil { + return "", nil, nil, err + } + + caFile, _, err = saveCertToFile(tempDir, caBytes, nil) + if err != nil { + return "", nil, nil, err + } + + for i, ip := range ips { + cert := &x509.Certificate{ + SerialNumber: big.NewInt(1001 + int64(i)), + Subject: pkix.Name{ + Organization: []string{"etcd"}, + OrganizationalUnit: []string{"etcd Security"}, + Locality: []string{"San Francisco"}, + Province: []string{"California"}, + Country: []string{"USA"}, + }, + IPAddresses: []net.IP{ip}, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(0, 0, 1), + SubjectKeyId: []byte{1, 2, 3, 4, 5}, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + KeyUsage: x509.KeyUsageDigitalSignature, + } + certKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return "", nil, nil, err + } + certBytes, err := x509.CreateCertificate(rand.Reader, cert, ca, &certKey.PublicKey, caKey) + if err != nil { + return "", nil, nil, err + } + certFile, keyFile, err := saveCertToFile(tempDir, certBytes, certKey) + if err != nil { + return "", nil, nil, err + } + certFiles = append(certFiles, certFile) + keyFiles = append(keyFiles, keyFile) + } + + return caFile, certFiles, keyFiles, nil +} + +func saveCertToFile(tempDir string, certBytes []byte, key *rsa.PrivateKey) (certFile string, keyFile string, err error) { + certPEM := new(bytes.Buffer) + pem.Encode(certPEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + cf, err := os.CreateTemp(tempDir, "*.crt") + if err != nil { + return "", "", err + } + defer cf.Close() + if _, err := cf.Write(certPEM.Bytes()); err != nil { + return "", "", err + } + + if key != nil { + certKeyPEM := new(bytes.Buffer) + pem.Encode(certKeyPEM, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(key), + }) + + kf, err := os.CreateTemp(tempDir, "*.key.insecure") + if err != nil { + return "", "", err + } + defer kf.Close() + if _, err := kf.Write(certKeyPEM.Bytes()); err != nil { + return "", "", err + } + + return cf.Name(), kf.Name(), nil + } + + return cf.Name(), "", nil +} + +func getLocalIP() (string, error) { + conn, err := net.Dial("udp", "8.8.8.8:80") + if err != nil { + return "", err + } + defer conn.Close() + + localAddress := conn.LocalAddr().(*net.UDPAddr) + + return localAddress.IP.String(), nil +}