Support setting LocalAddr in peer communication - with e2e tests

CHANGELOG/CHANGELOG-3.6.md

@@ -75,6 +75,7 @@ See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0...v3.6.0).
 - Decreased [`--snapshot-count` default value from 100,000 to 10,000](https://github.com/etcd-io/etcd/pull/15408)
 - Add [`etcd --tls-min-version --tls-max-version`](https://github.com/etcd-io/etcd/pull/15156) to enable support for TLS 1.3.
 - Add [quota to endpoint status response](https://github.com/etcd-io/etcd/pull/17877)
+- Add ['etcd --experimental-set-member-localaddr'](https://github.com/etcd-io/etcd/pull/17661) to enable using the first specified and non-loopback local address from initial-advertise-peer-urls as the local address when communicating with a peer.
 
 ### etcd grpc-proxy
 

client/pkg/transport/listener.go

@@ -193,6 +193,9 @@ type TLSInfo struct {
  // EmptyCN indicates that the cert must have empty CN.
  // If true, ClientConfig() will return an error for a cert with non empty CN.
  EmptyCN bool
+
+ // LocalAddr is the local IP address to use when communicating with a peer.
+ LocalAddr string
 }
 
 func (info TLSInfo) String() string {

client/pkg/transport/transport.go

@@ -30,10 +30,19 @@ func NewTransport(info TLSInfo, dialtimeoutd time.Duration) (*http.Transport, er
  return nil, err
  }
 
+ var ipAddr net.Addr
+ if info.LocalAddr != "" {
+ ipAddr, err = net.ResolveTCPAddr("tcp", info.LocalAddr+":0")
+ if err != nil {
+ return nil, err
+ }
+ }
+
  t := &http.Transport{
  Proxy: http.ProxyFromEnvironment,
  DialContext: (&net.Dialer{
- Timeout: dialtimeoutd,
+ Timeout: dialtimeoutd,
+ LocalAddr: ipAddr,
  // value taken from http.DefaultTransport
  KeepAlive: 30 * time.Second,
  }).DialContext,

server/config/config.go

@@ -204,6 +204,9 @@ type ServerConfig struct {
 
  // V2Deprecation defines a phase of v2store deprecation process.
  V2Deprecation V2DeprecationEnum `json:"v2-deprecation"`
+
+ // ExperimentalLocalAddress is the local IP address to use when communicating with a peer.
+ ExperimentalLocalAddress string `json:"experimental-local-address"`
 }
 
 // VerifyBootstrap sanity-checks the initial config for bootstrap case

server/embed/config.go

@@ -22,6 +22,7 @@ import (
  "math"
  "net"
  "net/http"
+ "net/netip"
  "net/url"
  "os"
  "path/filepath"
@@ -225,6 +226,12 @@ type Config struct {
  ClientAutoTLS bool
  PeerTLSInfo transport.TLSInfo
  PeerAutoTLS bool
+
+ // ExperimentalSetMemberLocalAddr enables using the first specified and
+ // non-loopback local address from initial-advertise-peer-urls as the local
+ // address when communicating with a peer.
+ ExperimentalSetMemberLocalAddr bool `json:"experimental-set-member-localaddr"`
+
  // SelfSignedCertValidity specifies the validity period of the client and peer certificates
  // that are automatically generated by etcd when you specify ClientAutoTLS and PeerAutoTLS,
  // the unit is year, and the default is 1
@@ -621,6 +628,8 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) {
  "initial-advertise-peer-urls",
  "List of this member's peer URLs to advertise to the rest of the cluster.",
  )
+ fs.BoolVar(&cfg.ExperimentalSetMemberLocalAddr, "experimental-set-member-localaddr", false, "Enable to have etcd use the first specified and non-loopback host from initial-advertise-peer-urls as the local address when communicating with a peer.")
+
  fs.Var(
  flags.NewUniqueURLsWithExceptions(DefaultAdvertiseClientURLs, ""),
  "advertise-client-urls",
@@ -1148,6 +1157,40 @@ func (cfg *Config) InitialClusterFromName(name string) (ret string) {
  return ret[1:]
 }
 
+// InferLocalAddr tries to determine the LocalAddr used when communicating with
+// an etcd peer. If SetMemberLocalAddr is true, then it will try to get the host
+// from AdvertisePeerUrls by searching for the first URL with a specified
+// non-loopback address. Otherwise, it defaults to empty string and the
+// LocalAddr used will be the default for the Golang HTTP client.
+func (cfg *Config) InferLocalAddr() string {
+ if !cfg.ExperimentalSetMemberLocalAddr {
+ return ""
+ }
+
+ lg := cfg.GetLogger()
+ lg.Info(
+ "searching for a suitable member local address in AdvertisePeerURLs",
+ zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerURLs()),
+ )
+ for _, peerURL := range cfg.AdvertisePeerUrls {
+ if addr, err := netip.ParseAddr(peerURL.Hostname()); err == nil {
+ if addr.IsLoopback() || addr.IsUnspecified() {
+ continue
+ }
+ lg.Info(
+ "setting member local address",
+ zap.String("LocalAddr", addr.String()),
+ )
+ return addr.String()
+ }
+ }
+ lg.Warn(
+ "unable to set a member local address due to lack of suitable local addresses",
+ zap.Strings("advertise-peer-urls", cfg.getAdvertisePeerURLs()),
+ )
+ return ""
+}
+
 func (cfg *Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateFlagNew }
 func (cfg *Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) }
 

server/embed/config_test.go

@@ -153,6 +153,138 @@ func TestUpdateDefaultClusterFromNameOverwrite(t *testing.T) {
  }
 }
 
+func TestInferLocalAddr(t *testing.T) {
+ tests := []struct {
+ name string
+ advertisePeerURLs []string
+ setMemberLocalAddr bool
+ expectedLocalAddr string
+ }{
+ {
+ "defaults, ExperimentalSetMemberLocalAddr=false ",
+ []string{DefaultInitialAdvertisePeerURLs},
+ false,
+ "",
+ },
+ {
+ "IPv4 address, ExperimentalSetMemberLocalAddr=false ",
+ []string{"https://192.168.100.110:2380"},
+ false,
+ "",
+ },
+ {
+ "defaults, ExperimentalSetMemberLocalAddr=true",
+ []string{DefaultInitialAdvertisePeerURLs},
+ true,
+ "",
+ },
+ {
+ "IPv4 unspecified address, ExperimentalSetMemberLocalAddr=true",
+ []string{"https://0.0.0.0:2380"},
+ true,
+ "",
+ },
+ {
+ "IPv6 unspecified address, ExperimentalSetMemberLocalAddr=true",
+ []string{"https://[::]:2380"},
+ true,
+ "",
+ },
+ {
+ "IPv4 loopback address, ExperimentalSetMemberLocalAddr=true",
+ []string{"https://127.0.0.1:2380"},
+ true,
+ "",
+ },
+ {
+ "IPv6 loopback address, ExperimentalSetMemberLocalAddr=true",
+ []string{"https://[::1]:2380"},
+ true,
+ "",
+ },
+ {
+ "IPv4 address, ExperimentalSetMemberLocalAddr=true",
+ []string{"https://192.168.100.110:2380"},
+ true,
+ "192.168.100.110",
+ },
+ {
+ "Hostname only, ExperimentalSetMemberLocalAddr=true",
+ []string{"https://123-host-3.corp.internal:2380"},
+ true,
+ "",
+ },
+ {
+ "Hostname and IPv4 address, ExperimentalSetMemberLocalAddr=true",
+ []string{"https://123-host-3.corp.internal:2380", "https://192.168.100.110:2380"},
+ true,
+ "192.168.100.110",
+ },
+ {
+ "IPv4 address and Hostname, ExperimentalSetMemberLocalAddr=true",
+ []string{"https://192.168.100.110:2380", "https://123-host-3.corp.internal:2380"},
+ true,
+ "192.168.100.110",
+ },
+ {
+ "IPv4 and IPv6 addresses, ExperimentalSetMemberLocalAddr=true",
+ []string{"https://192.168.100.110:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"},
+ true,
+ "192.168.100.110",
+ },
+ {
+ "IPv6 and IPv4 addresses, ExperimentalSetMemberLocalAddr=true",
+ // IPv4 addresses will always sort before IPv6 ones anyway
+ []string{"https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://192.168.100.110:2380"},
+ true,
+ "192.168.100.110",
+ },
+ {
+ "Hostname, IPv4 and IPv6 addresses, ExperimentalSetMemberLocalAddr=true",
+ []string{"https://123-host-3.corp.internal:2380", "https://192.168.100.110:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"},
+ true,
+ "192.168.100.110",
+ },
+ {
+ "Hostname, IPv6 and IPv4 addresses, ExperimentalSetMemberLocalAddr=true",
+ // IPv4 addresses will always sort before IPv6 ones anyway
+ []string{"https://123-host-3.corp.internal:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://192.168.100.110:2380"},
+ true,
+ "192.168.100.110",
+ },
+ {
+ "IPv6 address, ExperimentalSetMemberLocalAddr=true",
+ []string{"https://[2001:db8:85a3::8a2e:370:7334]:2380"},
+ true,
+ "2001:db8:85a3::8a2e:370:7334",
+ },
+ {
+ "Hostname and IPv6 address, ExperimentalSetMemberLocalAddr=true",
+ []string{"https://123-host-3.corp.internal:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"},
+ true,
+ "2001:db8:85a3::8a2e:370:7334",
+ },
+ {
+ "IPv6 address and Hostname, ExperimentalSetMemberLocalAddr=true",
+ []string{"https://[2001:db8:85a3::8a2e:370:7334]:2380", "https://123-host-3.corp.internal:2380"},
+ true,
+ "2001:db8:85a3::8a2e:370:7334",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ cfg := NewConfig()
+ cfg.AdvertisePeerUrls = types.MustNewURLs(tt.advertisePeerURLs)
+ cfg.ExperimentalSetMemberLocalAddr = tt.setMemberLocalAddr
+
+ require.NoError(t, cfg.Validate())
+ require.Equal(t, tt.expectedLocalAddr, cfg.InferLocalAddr())
+ })
+ }
+
+}
+
 func (s *securityConfig) equals(t *transport.TLSInfo) bool {
  return s.CertFile == t.CertFile &&
  s.CertAuth == t.ClientCertAuth &&

server/embed/etcd.go

@@ -227,6 +227,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
  ExperimentalBootstrapDefragThresholdMegabytes: cfg.ExperimentalBootstrapDefragThresholdMegabytes,
  ExperimentalMaxLearners: cfg.ExperimentalMaxLearners,
  V2Deprecation: cfg.V2DeprecationEffective(),
+ ExperimentalLocalAddress: cfg.InferLocalAddr(),
  }
 
  if srvcfg.ExperimentalEnableDistributedTracing {
@@ -245,6 +246,8 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
  )
  }
 
+ srvcfg.PeerTLSInfo.LocalAddr = srvcfg.ExperimentalLocalAddress
+
  print(e.cfg.logger, *cfg, srvcfg, memberInitialized)
 
  if e.Server, err = etcdserver.NewServer(srvcfg); err != nil {
@@ -336,6 +339,8 @@ func print(lg *zap.Logger, ec Config, sc config.ServerConfig, memberInitialized
  zap.Strings("advertise-client-urls", ec.getAdvertiseClientURLs()),
  zap.Strings("listen-client-urls", ec.getListenClientURLs()),
  zap.Strings("listen-metrics-urls", ec.getMetricsURLs()),
+ zap.Bool("experimental-set-member-localaddr", ec.ExperimentalSetMemberLocalAddr),
+ zap.String("experimental-local-address", sc.ExperimentalLocalAddress),
  zap.Strings("cors", cors),
  zap.Strings("host-whitelist", hss),
  zap.String("initial-cluster", sc.InitialPeerURLsMap.String()),

server/etcdmain/help.go

@@ -107,6 +107,8 @@ Member:
 Clustering:
  --initial-advertise-peer-urls 'http://localhost:2380'
  List of this member's peer URLs to advertise to the rest of the cluster.
+ --experimental-set-member-localaddr 'false'
+ Enable using the first specified and non-loopback local address from initial-advertise-peer-urls as the local address when communicating with a peer.
  --initial-cluster 'default=http://localhost:2380'
  Initial cluster configuration for bootstrapping.
  --initial-cluster-state 'new'