Modify URL parsing for advertise-urls used by etcd sidecar (#715)

* Use proper URLs for initial-advertise-peer-urls & advertise-client-urls instead of @ separator

* Use list for advertise-client-urls & initial-advertise-peer-urls

chart/etcd-backup-restore/templates/etcd-configmap.yaml

@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ .Release.Name }}-etcd-bootstrap
+  name: {{ .Release.Name }}-etcd-config
   namespace: {{ .Release.Namespace }}
   labels:
     app.kubernetes.io/name: etcd

chart/etcd-backup-restore/templates/etcd-statefulset.yaml

@@ -314,7 +314,7 @@ spec:
       volumes:
       - name: etcd-config-file
         configMap:
-          name: {{ .Release.Name }}-etcd-bootstrap
+          name: {{ .Release.Name }}-etcd-config
           defaultMode: 0644
           items:
           - key: etcd.conf.yaml

pkg/member/member_control.go

@@ -116,7 +116,7 @@ func NewMemberControl(etcdConnConfig *brtypes.EtcdConnectionConfig) Control {
 // AddMemberAsLearner add a member as a learner to the etcd cluster
 func (m *memberControl) AddMemberAsLearner(ctx context.Context) error {
 	//Add member as learner to cluster
-	memberURL, err := miscellaneous.GetMemberPeerURL(m.configFile, m.podName)
+	memberPeerURLs, err := miscellaneous.GetMemberPeerURLs(m.configFile)
 	if err != nil {
 		m.logger.Fatalf("Error fetching etcd member URL : %v", err)
 	}
@@ -130,10 +130,10 @@ func (m *memberControl) AddMemberAsLearner(ctx context.Context) error {
 	memAddCtx, cancel := context.WithTimeout(ctx, EtcdTimeout)
 	defer cancel()
 	start := time.Now()
-	response, err := cli.MemberAddAsLearner(memAddCtx, []string{memberURL})
+	response, err := cli.MemberAddAsLearner(memAddCtx, memberPeerURLs)
 	if err != nil {
 		if errors.Is(err, rpctypes.Error(rpctypes.ErrGRPCPeerURLExist)) || errors.Is(err, rpctypes.Error(rpctypes.ErrGRPCMemberExist)) {
-			m.logger.Infof("Member %s already part of etcd cluster", memberURL)
+			m.logger.Infof("Member %s with peer urls %v already part of etcd cluster", m.podName, memberPeerURLs)
 			return nil
 		} else if errors.Is(err, rpctypes.Error(rpctypes.ErrGRPCTooManyLearners)) {
 			m.logger.Infof("Unable to add member %s as a learner because the cluster already has a learner", m.podName)
@@ -205,16 +205,15 @@ func (m *memberControl) IsMemberInCluster(ctx context.Context) (bool, error) {
 func (m *memberControl) doUpdateMemberPeerAddress(ctx context.Context, cli etcdClient.ClusterCloser, id uint64) error {
 	// Already existing clusters or cluster after restoration have `http://localhost:2380` as the peer address. This needs to explicitly updated to the correct peer address.
 	m.logger.Infof("Updating member peer URL for %s", m.podName)
-
-	memberPeerURL, err := miscellaneous.GetMemberPeerURL(m.configFile, m.podName)
+	memberPeerURLs, err := miscellaneous.GetMemberPeerURLs(m.configFile)
 	if err != nil {
 		return fmt.Errorf("could not fetch member URL : %v", err)
 	}
 
 	memberUpdateCtx, cancel := context.WithTimeout(ctx, EtcdTimeout)
 	defer cancel()
 
-	if _, err = cli.MemberUpdate(memberUpdateCtx, id, []string{memberPeerURL}); err == nil {
+	if _, err = cli.MemberUpdate(memberUpdateCtx, id, memberPeerURLs); err == nil {
 		m.logger.Info("Successfully updated the member peer URL")
 		return nil
 	}

pkg/member/member_control_test.go

@@ -47,20 +47,24 @@ var _ = Describe("Membercontrol", func() {
 
 		outfile := "/tmp/etcd.conf.yaml"
 		etcdConfigYaml := `# Human-readable name for this member.
-    name: etcd1
-    data-dir: ` + os.Getenv("ETCD_DATA_DIR") + `
-    metrics: extensive
-    snapshot-count: 75000
-    enable-v2: false
-    quota-backend-bytes: 1073741824
-    listen-client-urls: http://0.0.0.0:2379
-    advertise-client-urls: http://0.0.0.0:2379
-    initial-advertise-peer-urls: http@etcd-main-peer@default@2380
-    initial-cluster: etcd1=http://0.0.0.0:2380
-    initial-cluster-token: new
-    initial-cluster-state: new
-    auto-compaction-mode: periodic
-    auto-compaction-retention: 30m`
+name: etcd1
+data-dir: ` + os.Getenv("ETCD_DATA_DIR") + `
+metrics: extensive
+snapshot-count: 75000
+enable-v2: false
+quota-backend-bytes: 1073741824
+listen-client-urls: http://0.0.0.0:2379
+advertise-client-urls:
+  ` + podName + `:
+    - http://0.0.0.0:2379
+initial-advertise-peer-urls:
+  ` + podName + `:
+    - http://etcd-main-peer.default:2380
+initial-cluster: etcd1=http://0.0.0.0:2380
+initial-cluster-token: new
+initial-cluster-state: new
+auto-compaction-mode: periodic
+auto-compaction-retention: 30m`
 
 		err := os.WriteFile(outfile, []byte(etcdConfigYaml), 0755)
 		Expect(err).ShouldNot(HaveOccurred())

pkg/miscellaneous/miscellaneous.go

@@ -62,6 +62,11 @@ const (
 	etcdWrapperPort = "9095"
 )
 
+type advertiseURLsConfig struct {
+	AdvertiseClientURLs      map[string][]string `json:"advertise-client-urls"`
+	InitialAdvertisePeerURLs map[string][]string `json:"initial-advertise-peer-urls"`
+}
+
 // GetLatestFullSnapshotAndDeltaSnapList returns the latest snapshot
 func GetLatestFullSnapshotAndDeltaSnapList(store brtypes.SnapStore) (*brtypes.Snapshot, brtypes.SnapList, error) {
 	var (
@@ -535,42 +540,88 @@ func ReadConfigFileAsMap(path string) (map[string]interface{}, error) {
 	return c, nil
 }
 
-// ParsePeerURL forms a PeerURL, given podName by parsing the initial-advertise-peer-urls
-func ParsePeerURL(initialAdvertisePeerURLs, podName string) (string, error) {
-	tokens := strings.Split(initialAdvertisePeerURLs, "@")
-	if len(tokens) < 4 {
-		return "", fmt.Errorf("invalid peer URL : %s", initialAdvertisePeerURLs)
+// parseAdvertiseURLsConfig reads and parses the config file and returns the advertise URLs config.
+func parseAdvertiseURLsConfig(configFile string) (*advertiseURLsConfig, error) {
+	var advURLsConfig advertiseURLsConfig
+	config, err := os.ReadFile(configFile)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read etcd config file at path: %s : %w", configFile, err)
+	}
+
+	if err := yaml.Unmarshal(config, &advURLsConfig); err != nil {
+		return nil, fmt.Errorf("unable to unmarshal etcd config yaml file at path: %s : %w", configFile, err)
 	}
-	domaiName := fmt.Sprintf("%s.%s.%s", tokens[1], tokens[2], "svc")
-	return fmt.Sprintf("%s://%s.%s:%s", tokens[0], podName, domaiName, tokens[3]), nil
+	return &advURLsConfig, nil
 }
 
-// IsPeerURLTLSEnabled checks whether the peer address is TLS enabled or not.
-func IsPeerURLTLSEnabled() (bool, error) {
-	podName, err := GetEnvVarOrError("POD_NAME")
+// GetMemberPeerURLs retrieves the initial advertise peer URLs for the etcd member using the POD_NAME environment variable.
+func GetMemberPeerURLs(configFile string) ([]string, error) {
+	memberName, err := GetEnvVarOrError("POD_NAME")
 	if err != nil {
-		return false, err
+		return nil, err
 	}
 
-	configFile := GetConfigFilePath()
+	advURLsConfig, err := parseAdvertiseURLsConfig(configFile)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse advertise URLs config: %w", err)
+	}
 
-	config, err := ReadConfigFileAsMap(configFile)
+	peerURLs, ok := advURLsConfig.InitialAdvertisePeerURLs[memberName]
+	if !ok || len(peerURLs) == 0 {
+		return nil, fmt.Errorf("no peer URLs found for pod %s", memberName)
+	}
+
+	for _, peerURL := range peerURLs {
+		if _, err := url.Parse(peerURL); err != nil {
+			return nil, fmt.Errorf("invalid peer URL %s: %w", peerURL, err)
+		}
+	}
+	return peerURLs, nil
+}
+
+// GetMemberClientURLs retrieves the advertise client URLs for the etcd member using the POD_NAME environment variable.
+func GetMemberClientURLs(configFile string) ([]string, error) {
+	memberName, err := GetEnvVarOrError("POD_NAME")
 	if err != nil {
-		return false, err
+		return nil, err
 	}
-	initAdPeerURL := config["initial-advertise-peer-urls"]
 
-	memberPeerURL, err := ParsePeerURL(initAdPeerURL.(string), podName)
+	advURLsConfig, err := parseAdvertiseURLsConfig(configFile)
 	if err != nil {
-		return false, err
+		return nil, fmt.Errorf("failed to parse advertise URLs config: %w", err)
+	}
+
+	clientURLs, ok := advURLsConfig.AdvertiseClientURLs[memberName]
+	if !ok || len(clientURLs) == 0 {
+		return nil, fmt.Errorf("no client URLs found for pod %s", memberName)
 	}
 
-	peerURL, err := url.Parse(memberPeerURL)
+	for _, clientURL := range clientURLs {
+		if _, err := url.Parse(clientURL); err != nil {
+			return nil, fmt.Errorf("invalid client URL %s: %w", clientURL, err)
+		}
+	}
+	return clientURLs, nil
+}
+
+// IsPeerURLTLSEnabled checks whether all peer URLs are TLS-enabled (i.e., use the "https" scheme).
+func IsPeerURLTLSEnabled() (bool, error) {
+	memberPeerURLs, err := GetMemberPeerURLs(GetConfigFilePath())
 	if err != nil {
-		return false, err
+		return false, fmt.Errorf("failed to get initial advertise peer URLs: %w", err)
+	}
+
+	for _, peerURL := range memberPeerURLs {
+		parsedPeerURL, err := url.Parse(peerURL)
+		if err != nil {
+			return false, fmt.Errorf("failed to parse peer URL %s: %w", peerURL, err)
+		}
+		if parsedPeerURL.Scheme != https {
+			return false, nil
+		}
 	}
 
-	return peerURL.Scheme == https, nil
+	return true, nil
 }
 
 // GetPrevScheduledSnapTime returns the previous schedule snapshot time.
@@ -611,23 +662,6 @@ func RemoveDir(dir string) error {
 	return nil
 }
 
-// GetMemberPeerURL returns the peerURL from fiven configuration file provided to etcd member.
-func GetMemberPeerURL(configFile string, podName string) (string, error) {
-	config, err := ReadConfigFileAsMap(configFile)
-	if err != nil {
-		return "", err
-	}
-	initAdPeerURL := config["initial-advertise-peer-urls"]
-	if initAdPeerURL == nil {
-		return "", fmt.Errorf("initial-advertise-peer-urls must be set in etcd config")
-	}
-	peerURL, err := ParsePeerURL(initAdPeerURL.(string), podName)
-	if err != nil {
-		return "", fmt.Errorf("could not parse peer URL from the config file : %v", err)
-	}
-	return peerURL, nil
-}
-
 // RestartEtcdWrapper is to call the "/stop" endpoint of etcd-wrapper to restart the etcd-wrapper container.
 func RestartEtcdWrapper(ctx context.Context, tlsEnabled bool, etcdConnectionConfig *brtypes.EtcdConnectionConfig) error {
 	client := &http.Client{}