Implement flag --set-member-localaddr

Which sets the LocalAddr to an IP address from --initial-advertise-peer-urls.

Also adds e2e test that requires this flag to succeed.

Signed-off-by: Edwin Xie <[email protected]>

client/pkg/transport/listener.go

@@ -194,7 +194,7 @@ type TLSInfo struct {
  // If true, ClientConfig() will return an error for a cert with non empty CN.
  EmptyCN bool
 
- // LocalAddr is the local IP address to use when communicating peer.
+ // LocalAddr is the local IP address to use when communicating with a peer.
  LocalAddr string
 }
 

server/config/config.go

@@ -205,7 +205,7 @@ type ServerConfig struct {
  // V2Deprecation defines a phase of v2store deprecation process.
  V2Deprecation V2DeprecationEnum `json:"v2-deprecation"`
 
- // LocalAddr is the local IP address to use when communicating peer.
+ // LocalAddr is the local IP address to use when communicating with a peer.
  LocalAddr string `json:"local-address"`
 }
 

server/embed/config.go

@@ -22,6 +22,7 @@ import (
  "math"
  "net"
  "net/http"
+ "net/netip"
  "net/url"
  "os"
  "path/filepath"
@@ -220,6 +221,12 @@ type Config struct {
  ClientAutoTLS bool
  PeerTLSInfo transport.TLSInfo
  PeerAutoTLS bool
+
+ // SetMemberLocalAddr specifies if true, that etcd should use the first
+ // specified and non-loopback host in AdvertisePeerUrls as the LocalAddr when
+ // communicating with a peer.
+ SetMemberLocalAddr bool `json:"set-member-localaddr"`
+
  // SelfSignedCertValidity specifies the validity period of the client and peer certificates
  // that are automatically generated by etcd when you specify ClientAutoTLS and PeerAutoTLS,
  // the unit is year, and the default is 1
@@ -327,9 +334,6 @@ type Config struct {
  // AuthTokenTTL in seconds of the simple token
  AuthTokenTTL uint `json:"auth-token-ttl"`
 
- // PeerLocalAddr is the local IP address to use when communicating peer.
- PeerLocalAddr string `json:"peer-local-addr"`
-
  ExperimentalInitialCorruptCheck bool `json:"experimental-initial-corrupt-check"`
  ExperimentalCorruptCheckTime time.Duration `json:"experimental-corrupt-check-time"`
  ExperimentalCompactHashCheckEnabled bool `json:"experimental-compact-hash-check-enabled"`
@@ -614,6 +618,8 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) {
  "initial-advertise-peer-urls",
  "List of this member's peer URLs to advertise to the rest of the cluster.",
  )
+ fs.BoolVar(&cfg.SetMemberLocalAddr, "set-member-localaddr", false, "Enable to have etcd use the first specified and non-loopback host from initial-advertise-peer-urls as the local address when communicating with a peer.")
+
  fs.Var(
  flags.NewUniqueURLsWithExceptions(DefaultAdvertiseClientURLs, ""),
  "advertise-client-urls",
@@ -665,8 +671,6 @@ func (cfg *Config) AddFlags(fs *flag.FlagSet) {
  fs.StringVar(&cfg.PeerTLSInfo.ClientKeyFile, "peer-client-key-file", "", "Path to an explicit peer client TLS key file otherwise peer key file will be used when client auth is required.")
  fs.BoolVar(&cfg.PeerTLSInfo.ClientCertAuth, "peer-client-cert-auth", false, "Enable peer client cert authentication.")
  fs.StringVar(&cfg.PeerTLSInfo.TrustedCAFile, "peer-trusted-ca-file", "", "Path to the peer server TLS trusted CA file.")
- fs.StringVar(&cfg.PeerTLSInfo.LocalAddr, "peer-local-addr", "", "peer-local-addr is the local IP address to use when communicating peer.")
-
  fs.BoolVar(&cfg.PeerAutoTLS, "peer-auto-tls", false, "Peer TLS using generated certificates")
  fs.UintVar(&cfg.SelfSignedCertValidity, "self-signed-cert-validity", 1, "The validity period of the client and peer certificates, unit is year")
  fs.StringVar(&cfg.PeerTLSInfo.CRLFile, "peer-crl-file", "", "Path to the peer certificate revocation list file.")
@@ -1141,6 +1145,26 @@ func (cfg *Config) InitialClusterFromName(name string) (ret string) {
  return ret[1:]
 }
 
+// InferLocalAddr tries to determine the LocalAddr used when communicating with
+// an etcd peer. If SetMemberLocalAddr is true, then it will try to get the host
+// from AdvertisePeerUrls by searching for the first URL with a specified
+// non-loopback address. Otherwise, it defaults to empty string and the
+// LocalAddr used will be the default for the Golang HTTP client.
+func (cfg *Config) InferLocalAddr() string {
+ if !cfg.SetMemberLocalAddr {
+ return ""
+ }
+ for _, peerUrl := range cfg.AdvertisePeerUrls {
+ if addr, err := netip.ParseAddr(peerUrl.Hostname()); err == nil {
+ if addr.IsLoopback() || addr.IsUnspecified() {
+ continue
+ }
+ return addr.String()
+ }
+ }
+ return ""
+}
+
 func (cfg *Config) IsNewCluster() bool { return cfg.ClusterState == ClusterStateFlagNew }
 func (cfg *Config) ElectionTicks() int { return int(cfg.ElectionMs / cfg.TickMs) }
 

server/embed/config_test.go

@@ -152,6 +152,102 @@ func TestUpdateDefaultClusterFromNameOverwrite(t *testing.T) {
  }
 }
 
+func TestInferLocalAddr(t *testing.T) {
+ tests := []struct {
+ name string
+ advertisePeerUrls []string
+ setMemberLocalAddr bool
+ expectedLocalAddr string
+ }{
+ {
+ "defaults, SetMemberLocalAddr=false ",
+ []string{DefaultInitialAdvertisePeerURLs},
+ false,
+ "",
+ },
+ {
+ "IPv4 address, SetMemberLocalAddr=false ",
+ []string{"https://192.168.100.110:2380"},
+ false,
+ "",
+ },
+ {
+ "defaults, SetMemberLocalAddr=true",
+ []string{DefaultInitialAdvertisePeerURLs},
+ true,
+ "",
+ },
+ {
+ "IPv4 unspecified address, SetMemberLocalAddr=true",
+ []string{"https://0.0.0.0:2380"},
+ true,
+ "",
+ },
+ {
+ "IPv6 unspecified address, SetMemberLocalAddr=true",
+ []string{"https://[::]:2380"},
+ true,
+ "",
+ },
+ {
+ "IPv4 loopback address, SetMemberLocalAddr=true",
+ []string{"https://127.0.0.1:2380"},
+ true,
+ "",
+ },
+ {
+ "IPv6 loopback address, SetMemberLocalAddr=true",
+ []string{"https://[::1]:2380"},
+ true,
+ "",
+ },
+ {
+ "IPv4 address, SetMemberLocalAddr=true",
+ []string{"https://192.168.100.110:2380"},
+ true,
+ "192.168.100.110",
+ },
+ {
+ "Hostname and IPv4 address, SetMemberLocalAddr=true",
+ []string{"https://123-host-3.corp.internal:2380", "https://192.168.100.110:2380"},
+ true,
+ "192.168.100.110",
+ },
+ {
+ "IPv4 and IPv6 address, SetMemberLocalAddr=true",
+ // IPv4 addresses will always sort before IPv6 ones anyways
+ []string{"https://192.168.100.110:2380", "https://[2001:db8:85a3::8a2e:370:7334]:2380"},
+ true,
+ "192.168.100.110",
+ },
+ {
+ "IPv6 address, SetMemberLocalAddr=true",
+ []string{"https://[2001:db8:85a3::8a2e:370:7334]:2380"},
+ true,
+ "2001:db8:85a3::8a2e:370:7334",
+ },
+ }
+
+ for i, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ cfg := NewConfig()
+ cfg.AdvertisePeerUrls = types.MustNewURLs(tt.advertisePeerUrls)
+ cfg.SetMemberLocalAddr = tt.setMemberLocalAddr
+
+ if err := cfg.Validate(); err != nil {
+ t.Errorf("#%d: failed to validate test Config: %v", i, err)
+ return
+ }
+
+ inferredLocalAddr := cfg.InferLocalAddr()
+ if inferredLocalAddr != tt.expectedLocalAddr {
+ t.Errorf("#%d: LocalAddr = %s, want = %s", i, inferredLocalAddr, tt.expectedLocalAddr)
+ }
+ })
+ }
+
+}
+
 func (s *securityConfig) equals(t *transport.TLSInfo) bool {
  return s.CertFile == t.CertFile &&
  s.CertAuth == t.ClientCertAuth &&

server/embed/etcd.go

@@ -227,6 +227,7 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
  ExperimentalBootstrapDefragThresholdMegabytes: cfg.ExperimentalBootstrapDefragThresholdMegabytes,
  ExperimentalMaxLearners: cfg.ExperimentalMaxLearners,
  V2Deprecation: cfg.V2DeprecationEffective(),
+ LocalAddr: cfg.InferLocalAddr(),
  }
 
  if srvcfg.ExperimentalEnableDistributedTracing {
@@ -245,6 +246,8 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) {
  )
  }
 
+ srvcfg.PeerTLSInfo.LocalAddr = srvcfg.LocalAddr
+
  print(e.cfg.logger, *cfg, srvcfg, memberInitialized)
 
  if e.Server, err = etcdserver.NewServer(srvcfg); err != nil {
@@ -336,6 +339,8 @@ func print(lg *zap.Logger, ec Config, sc config.ServerConfig, memberInitialized
  zap.Strings("advertise-client-urls", ec.getAdvertiseClientUrls()),
  zap.Strings("listen-client-urls", ec.getListenClientUrls()),
  zap.Strings("listen-metrics-urls", ec.getMetricsURLs()),
+ zap.Bool("set-member-localaddr", ec.SetMemberLocalAddr),
+ zap.String("local-addr", sc.LocalAddr),
  zap.Strings("cors", cors),
  zap.Strings("host-whitelist", hss),
  zap.String("initial-cluster", sc.InitialPeerURLsMap.String()),

server/etcdmain/help.go

@@ -107,6 +107,8 @@ Member:
 Clustering:
  --initial-advertise-peer-urls 'http://localhost:2380'
  List of this member's peer URLs to advertise to the rest of the cluster.
+ --set-member-localaddr 'false'
+ Enable to have etcd use the first specified and non-loopback host from initial-advertise-peer-urls as the local address when communicating with a peer.
  --initial-cluster 'default=http://localhost:2380'
  Initial cluster configuration for bootstrapping.
  --initial-cluster-state 'new'
@@ -221,9 +223,6 @@ Security:
  Minimum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3.
  --tls-max-version ''
  Maximum TLS version supported by etcd. Possible values: TLS1.2, TLS1.3 (empty will be auto-populated by Go).
- --peer-local-addr ''
- LocalAddr is the local IP address to use when communicating peer.
-
 
 Auth:
  --auth-token 'simple'

tests/e2e/etcd_config_test.go

@@ -17,6 +17,7 @@ package e2e
 import (
  "context"
  "fmt"
+ "net"
  "os"
  "strings"
  "testing"
@@ -274,6 +275,107 @@ func TestEtcdPeerNameAuth(t *testing.T) {
  }
 }
 
+// TestEtcdPeerLocalAddr checks that the inter peer auth works with when
+// the member LocalAddr is set.
+func TestEtcdPeerLocalAddr(t *testing.T) {
+ e2e.SkipInShortMode(t)
+
+ nodeIP, err := getLocalIP()
+ t.Log("Using node IP", nodeIP)
+ require.NoError(t, err)
+
+ peers, tmpdirs := make([]string, 3), make([]string, 3)
+
+ for i := range peers {
+ peerIP := nodeIP
+ if i == 0 {
+ peerIP = "127.0.0.1"
+ }
+ peers[i] = fmt.Sprintf("e%d=https://%s:%d", i, peerIP, e2e.EtcdProcessBasePort+i)
+ tmpdirs[i] = t.TempDir()
+ }
+ procs := make([]*expect.ExpectProcess, len(peers))
+ defer func() {
+ for i := range procs {
+ if procs[i] != nil {
+ procs[i].Stop()
+ procs[i].Close()
+ }
+ os.RemoveAll(tmpdirs[i])
+ }
+ }()
+
+ tempDir := t.TempDir()
+ caFile, certFiles, keyFiles, err := generateCertsForIPs(tempDir, []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP(nodeIP)})
+ require.NoError(t, err)
+
+ defer func() {
+ os.RemoveAll(tempDir)
+ }()
+
+ // node 0 (127.0.0.1) does not set localaddr, while nodes 1 and nodes 2 (both use host's IP) do.
+ // The other two nodes will reject connections from node 0 warning that node 0's certificate is valid only for
+ // 127.0.0.1, not the host IP, since node 0 will try to connect to the other peers with the host IP
+ // as the client address.
+ // Node 0 will not reject connections from the other nodes since they will
+ // use the host's IP to connect (due to --set-member-localaddr)
+ for i := range procs {
+ peerIP := nodeIP
+ if i == 0 {
+ peerIP = "127.0.0.1"
+ }
+ ic := strings.Join(peers, ",")
+ commonArgs := []string{
+ e2e.BinPath.Etcd,
+ "--name", fmt.Sprintf("e%d", i),
+ "--listen-client-urls", "http://0.0.0.0:0",
+ "--data-dir", tmpdirs[i],
+ "--advertise-client-urls", "http://0.0.0.0:0",
+ "--initial-advertise-peer-urls", fmt.Sprintf("https://%s:%d", peerIP, e2e.EtcdProcessBasePort+i),
+ "--listen-peer-urls", fmt.Sprintf("https://%s:%d,https://%s:%d", peerIP, e2e.EtcdProcessBasePort+i, peerIP, e2e.EtcdProcessBasePort+len(peers)+i),
+ "--initial-cluster", ic,
+ }
+
+ var args []string
+ if i == 0 {
+ args = []string{
+ "--peer-cert-file", certFiles[0],
+ "--peer-key-file", keyFiles[0],
+ "--peer-trusted-ca-file", caFile,
+ "--peer-client-cert-auth",
+ }
+ } else {
+ args = []string{
+ "--peer-cert-file", certFiles[1],
+ "--peer-key-file", keyFiles[1],
+ "--peer-trusted-ca-file", caFile,
+ "--peer-client-cert-auth",
+ "--set-member-localaddr",
+ }
+ }
+
+ commonArgs = append(commonArgs, args...)
+
+ p, err := e2e.SpawnCmd(commonArgs, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ procs[i] = p
+ }
+
+ for i, p := range procs {
+ var expect []string
+ if i == 0 {
+ expect = e2e.EtcdServerReadyLines
+ } else {
+ expect = []string{"x509: certificate is valid for 127.0.0.1, not "}
+ }
+ if err := e2e.WaitReadyExpectProc(context.TODO(), p, expect); err != nil {
+ t.Fatal(err)
+ }
+ }
+}
+
 func TestGrpcproxyAndCommonName(t *testing.T) {
  e2e.SkipInShortMode(t)