Merge pull request etcd-io#17661 from flawedmatrix/fix/17068

Support setting LocalAddr in peer communication - with e2e tests

CHANGELOG/CHANGELOG-3.6.md

@@ -75,6 +75,7 @@ See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0...v3.6.0).
 - Decreased [`--snapshot-count` default value from 100,000 to 10,000](https://github.com/etcd-io/etcd/pull/15408)
 - Add [`etcd --tls-min-version --tls-max-version`](https://github.com/etcd-io/etcd/pull/15156) to enable support for TLS 1.3.
 - Add [quota to endpoint status response](https://github.com/etcd-io/etcd/pull/17877)
+- Add ['etcd --experimental-set-member-localaddr'](https://github.com/etcd-io/etcd/pull/17661) to enable using the first specified and non-loopback local address from initial-advertise-peer-urls as the local address when communicating with a peer.
 
 ### etcd grpc-proxy
 

client/pkg/transport/listener.go

@@ -193,6 +193,9 @@ type TLSInfo struct {
 	// EmptyCN indicates that the cert must have empty CN.
 	// If true, ClientConfig() will return an error for a cert with non empty CN.
 	EmptyCN bool
+
+	// LocalAddr is the local IP address to use when communicating with a peer.
+	LocalAddr string
 }
 
 func (info TLSInfo) String() string {

client/pkg/transport/transport.go

@@ -30,10 +30,19 @@ func NewTransport(info TLSInfo, dialtimeoutd time.Duration) (*http.Transport, er
 		return nil, err
 	}
 
+	var ipAddr net.Addr
+	if info.LocalAddr != "" {
+		ipAddr, err = net.ResolveTCPAddr("tcp", info.LocalAddr+":0")
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	t := &http.Transport{
 		Proxy: http.ProxyFromEnvironment,
 		DialContext: (&net.Dialer{
-			Timeout: dialtimeoutd,
+			Timeout:   dialtimeoutd,
+			LocalAddr: ipAddr,
 			// value taken from http.DefaultTransport
 			KeepAlive: 30 * time.Second,
 		}).DialContext,

server/config/config.go

@@ -204,6 +204,9 @@ type ServerConfig struct {
 
 	// V2Deprecation defines a phase of v2store deprecation process.
 	V2Deprecation V2DeprecationEnum `json:"v2-deprecation"`
+
+	// ExperimentalLocalAddress is the local IP address to use when communicating with a peer.
+	ExperimentalLocalAddress string `json:"experimental-local-address"`
 }
 
 // VerifyBootstrap sanity-checks the initial config for bootstrap case

server/embed/config.go

@@ -22,6 +22,7 @@ import (
 	"math"
 	"net"
 	"net/http"
+	"net/netip"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -225,6 +226,12 @@ type Config struct {
 	ClientAutoTLS bool
 	PeerTLSInfo   transport.TLSInfo
 	PeerAutoTLS   bool
+
+	// ExperimentalSetMemberLocalAddr enables using the first specified and
+	// non-loopback local address from initial-advertise-peer-urls as the local
+	// address when communicating with a peer.
+	ExperimentalSetMemberLocalAddr bool `json:"experimental-set-member-localaddr"`
+
 	// SelfSignedCertValidity specifies the validity period of the client and peer certificates
 	// that are automatically generated by etcd when you specify ClientAutoTLS and PeerAutoTLS,
 	// the unit is year, and the default is 1
@@ -621,6 +628,8 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) {
 		"initial-advertise-peer-urls",
 		"List of this member's peer URLs to advertise to the rest of the cluster.",
 	)
+	fs.BoolVar(&cfg.ExperimentalSetMemberLocalAddr, "experimental-set-member-localaddr", false, "Enable to have etcd use the first specified and non-loopback host from initial-advertise-peer-urls as the local address when communicating with a peer.")
+
 	fs.Var(
 		flags.NewUniqueURLsWithExceptions(DefaultAdvertiseClientURLs, ""),
 		"advertise-client-urls",
@@ -1148,6 +1157,40 @@ func (cfg *Config) InitialClusterFromName(name string) (ret string) {
 	return ret[1:]
 }
 
+// InferLocalAddr tries to determine the LocalAddr used when communicating with
+// an etcd peer. If SetMemberLocalAddr is true, then it will try to get the host
+// from AdvertisePeerUrls by searching for the first URL with a specified
+// non-loopback address. Otherwise, it defaults to empty string and the
+// LocalAddr used will be the default for the Golang HTTP client.
+func (cfg *Config) InferLocalAddr() string {
+	if !cfg.ExperimentalSetMemberLocalAddr {
+		return ""
+	}
+
+	lg := cfg.GetLogger()
+	lg.Info(
+		"searching for a suitable member local address in AdvertisePeerURLs",
+		zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerURLs()),
+	)
+	for _, peerURL := range cfg.AdvertisePeerUrls {
+		if addr, err := netip.ParseAddr(peerURL.Hostname()); err == nil {
+			if addr.IsLoopback() || addr.IsUnspecified() {
+				continue
+			}
+			lg.Info(
+				"setting member local address",
+				zap.String("LocalAddr", addr.String()),
+			)
+			return addr.String()
+		}
+	}
+	lg.Warn(
+		"unable to set a member local address due to lack of suitable local addresses",
+		zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerURLs()),
+	)
+	return ""
+}
+
 func (cfg *Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateFlagNew }
 func (cfg *Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) }
 

server/embed/config_test.go

@@ -153,6 +153,138 @@ func TestUpdateDefaultClusterFromNameOverwrite(t *testing.T) {
 	}
 }
 
+func TestInferLocalAddr(t *testing.T) {
+	tests := []struct {
+		name               string
+		advertisePeerURLs  []string
+		setMemberLocalAddr bool
+		expectedLocalAddr  string
+	}{
+		{
+			"defaults, ExperimentalSetMemberLocalAddr=false ",
+			[]string{DefaultInitialAdvertisePeerURLs},
+			false,
+			"",
+		},
+		{
+			"IPv4 address, ExperimentalSetMemberLocalAddr=false ",
+			[]string{"https://192.168.100.110:2380"},
+			false,
+			"",
+		},
+		{
+			"defaults, ExperimentalSetMemberLocalAddr=true",
+			[]string{DefaultInitialAdvertisePeerURLs},
+			true,
+			"",
+		},
+		{
+			"IPv4 unspecified address, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://0.0.0.0:2380"},
+			true,
+			"",
+		},
+		{
+			"IPv6 unspecified address, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://[::]:2380"},
+			true,
+			"",
+		},
+		{
+			"IPv4 loopback address, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://127.0.0.1:2380"},
+			true,
+			"",
+		},
+		{
+			"IPv6 loopback address, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://[::1]:2380"},
+			true,
+			"",
+		},
+		{
+			"IPv4 address, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://192.168.100.110:2380"},
+			true,
+			"192.168.100.110",
+		},
+		{
+			"Hostname only, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://123-host-3.corp.internal:2380"},
+			true,
+			"",
+		},
+		{
+			"Hostname and IPv4 address, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://123-host-3.corp.internal:2380", "https://192.168.100.110:2380"},
+			true,
+			"192.168.100.110",
+		},
+		{
+			"IPv4 address and Hostname, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://192.168.100.110:2380", "https://123-host-3.corp.internal:2380"},
+			true,
+			"192.168.100.110",
+		},
+		{
+			"IPv4 and IPv6 addresses, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://192.168.100.110:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"},
+			true,
+			"192.168.100.110",
+		},
+		{
+			"IPv6 and IPv4 addresses, ExperimentalSetMemberLocalAddr=true",
+			// IPv4 addresses will always sort before IPv6 ones anyway
+			[]string{"https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://192.168.100.110:2380"},
+			true,
+			"192.168.100.110",
+		},
+		{
+			"Hostname, IPv4 and IPv6 addresses, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://123-host-3.corp.internal:2380", "https://192.168.100.110:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"},
+			true,
+			"192.168.100.110",
+		},
+		{
+			"Hostname, IPv6 and IPv4 addresses, ExperimentalSetMemberLocalAddr=true",
+			// IPv4 addresses will always sort before IPv6 ones anyway
+			[]string{"https://123-host-3.corp.internal:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://192.168.100.110:2380"},
+			true,
+			"192.168.100.110",
+		},
+		{
+			"IPv6 address, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://[2001:db8:85a3::8a2e:370:7334]:2380"},
+			true,
+			"2001:db8:85a3::8a2e:370:7334",
+		},
+		{
+			"Hostname and IPv6 address, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://123-host-3.corp.internal:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"},
+			true,
+			"2001:db8:85a3::8a2e:370:7334",
+		},
+		{
+			"IPv6 address and Hostname, ExperimentalSetMemberLocalAddr=true",
+			[]string{"https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://123-host-3.corp.internal:2380"},
+			true,
+			"2001:db8:85a3::8a2e:370:7334",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cfg := NewConfig()
+			cfg.AdvertisePeerUrls = types.MustNewURLs(tt.advertisePeerURLs)
+			cfg.ExperimentalSetMemberLocalAddr = tt.setMemberLocalAddr
+
+			require.NoError(t, cfg.Validate())
+			require.Equal(t, tt.expectedLocalAddr, cfg.InferLocalAddr())
+		})
+	}
+
+}
+
 func (s *securityConfig) equals(t *transport.TLSInfo) bool {
 	return s.CertFile == t.CertFile &&
 		s.CertAuth == t.ClientCertAuth &&

server/embed/etcd.go

@@ -227,6 +227,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
 		ExperimentalBootstrapDefragThresholdMegabytes: cfg.ExperimentalBootstrapDefragThresholdMegabytes,
 		ExperimentalMaxLearners:                       cfg.ExperimentalMaxLearners,
 		V2Deprecation:                                 cfg.V2DeprecationEffective(),
+		ExperimentalLocalAddress:                      cfg.InferLocalAddr(),
 	}
 
 	if srvcfg.ExperimentalEnableDistributedTracing {
@@ -245,6 +246,8 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
 		)
 	}
 
+	srvcfg.PeerTLSInfo.LocalAddr = srvcfg.ExperimentalLocalAddress
+
 	print(e.cfg.logger, *cfg, srvcfg, memberInitialized)
 
 	if e.Server, err = etcdserver.NewServer(srvcfg); err != nil {
@@ -336,6 +339,8 @@ func print(lg *zap.Logger, ec Config, sc config.ServerConfig, memberInitialized
 		zap.Strings("advertise-client-urls", ec.getAdvertiseClientURLs()),
 		zap.Strings("listen-client-urls", ec.getListenClientURLs()),
 		zap.Strings("listen-metrics-urls", ec.getMetricsURLs()),
+		zap.Bool("experimental-set-member-localaddr", ec.ExperimentalSetMemberLocalAddr),
+		zap.String("experimental-local-address", sc.ExperimentalLocalAddress),
 		zap.Strings("cors", cors),
 		zap.Strings("host-whitelist", hss),
 		zap.String("initial-cluster", sc.InitialPeerURLsMap.String()),

server/etcdmain/help.go

@@ -107,6 +107,8 @@ Member:
 Clustering:
   --initial-advertise-peer-urls 'http://localhost:2380'
     List of this member's peer URLs to advertise to the rest of the cluster.
+  --experimental-set-member-localaddr 'false'
+    Enable using the first specified and non-loopback local address from initial-advertise-peer-urls as the local address when communicating with a peer.
   --initial-cluster 'default=http://localhost:2380'
     Initial cluster configuration for bootstrapping.
   --initial-cluster-state 'new'