From 8ff42362e6a3461f094c3af5392bcf51ce5cb750 Mon Sep 17 00:00:00 2001 From: Anvesh Reddy Pinnapureddy Date: Tue, 8 Oct 2024 00:01:02 +0530 Subject: [PATCH 1/8] rename configMap | add snapshotCount to spec.etcd to make the snapshot-count configurable | use proper url formatting for peer and client urls in etcd config --- api/v1alpha1/etcd.go | 4 + api/v1alpha1/etcd_test.go | 18 +-- api/v1alpha1/helper.go | 2 +- api/v1alpha1/helper_test.go | 2 +- api/v1alpha1/zz_generated.deepcopy.go | 5 + .../crd-druid.gardener.cloud_etcds.yaml | 6 + .../bases/crd-druid.gardener.cloud_etcds.yaml | 6 + ...m-permanent-quorum-loss-in-etcd-cluster.md | 131 ++++++++++++++++++ .../03-scaling-up-an-etcd-cluster.md | 2 +- .../component/configmap/configmap_test.go | 9 +- internal/component/configmap/etcdconfig.go | 49 +++++-- test/e2e/etcd_backup_test.go | 4 +- test/e2e/utils.go | 6 +- test/it/setup/setup.go | 4 +- test/utils/etcd.go | 2 + 15 files changed, 214 insertions(+), 36 deletions(-) create mode 100644 docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md diff --git a/api/v1alpha1/etcd.go b/api/v1alpha1/etcd.go index b079f16a4..8d311e95e 100644 --- a/api/v1alpha1/etcd.go +++ b/api/v1alpha1/etcd.go @@ -194,6 +194,10 @@ type EtcdConfig struct { // Quota defines the etcd DB quota. // +optional Quota *resource.Quantity `json:"quota,omitempty"` + // SnapshotCount defines the number of applied Raft entries to hold in-memory before compaction. + // More info: https://etcd.io/docs/v3.4/op-guide/maintenance/#raft-log-retention + // +optional + SnapshotCount *int64 `json:"snapshotCount,omitempty"` // DefragmentationSchedule defines the cron standard schedule for defragmentation of etcd. // +optional DefragmentationSchedule *string `json:"defragmentationSchedule,omitempty"` diff --git a/api/v1alpha1/etcd_test.go b/api/v1alpha1/etcd_test.go index 3d3651b12..adcb2cd3d 100644 --- a/api/v1alpha1/etcd_test.go +++ b/api/v1alpha1/etcd_test.go @@ -118,10 +118,11 @@ func TestIsReconciliationInProgress(t *testing.T) { func createEtcd(name, namespace string) *Etcd { var ( - clientPort int32 = 2379 - serverPort int32 = 2380 - backupPort int32 = 8080 - metricLevel = Basic + clientPort int32 = 2379 + serverPort int32 = 2380 + backupPort int32 = 8080 + metricLevel = Basic + snapshotCount int64 = 75000 ) garbageCollectionPeriod := metav1.Duration{ @@ -238,10 +239,11 @@ func createEtcd(name, namespace string) *Etcd { "memory": resource.MustParse("1000Mi"), }, }, - ClientPort: &clientPort, - ServerPort: &serverPort, - ClientUrlTLS: clientTlsConfig, - PeerUrlTLS: peerTlsConfig, + ClientPort: &clientPort, + ServerPort: &serverPort, + SnapshotCount: &snapshotCount, + ClientUrlTLS: clientTlsConfig, + PeerUrlTLS: peerTlsConfig, }, }, } diff --git a/api/v1alpha1/helper.go b/api/v1alpha1/helper.go index a6bb27a3e..068c34f70 100644 --- a/api/v1alpha1/helper.go +++ b/api/v1alpha1/helper.go @@ -31,7 +31,7 @@ func GetServiceAccountName(etcdObjMeta metav1.ObjectMeta) string { // GetConfigMapName returns the name of the configmap for the Etcd. func GetConfigMapName(etcdObjMeta metav1.ObjectMeta) string { - return fmt.Sprintf("etcd-bootstrap-%s", string(etcdObjMeta.UID[:6])) + return fmt.Sprintf("%s-config-%s", etcdObjMeta.Name, (etcdObjMeta.UID[:6])) } // GetCompactionJobName returns the compaction job name for the Etcd. diff --git a/api/v1alpha1/helper_test.go b/api/v1alpha1/helper_test.go index cc9776279..a35fbabe7 100644 --- a/api/v1alpha1/helper_test.go +++ b/api/v1alpha1/helper_test.go @@ -54,7 +54,7 @@ func TestGetConfigMapName(t *testing.T) { uid := uuid.NewUUID() etcdObjMeta := createEtcdObjectMetadata(uid, nil, nil, false) configMapName := GetConfigMapName(etcdObjMeta) - g.Expect(configMapName).To(Equal("etcd-bootstrap-" + string(uid[:6]))) + g.Expect(configMapName).To(Equal(etcdObjMeta.Name + "-config-" + string(uid[:6]))) } func TestGetCompactionJobName(t *testing.T) { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index a94f34e03..72bc071d8 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -235,6 +235,11 @@ func (in *EtcdConfig) DeepCopyInto(out *EtcdConfig) { x := (*in).DeepCopy() *out = &x } + if in.SnapshotCount != nil { + in, out := &in.SnapshotCount, &out.SnapshotCount + *out = new(int64) + **out = **in + } if in.DefragmentationSchedule != nil { in, out := &in.DefragmentationSchedule, &out.DefragmentationSchedule *out = new(string) diff --git a/charts/druid/charts/crds/templates/crd-druid.gardener.cloud_etcds.yaml b/charts/druid/charts/crds/templates/crd-druid.gardener.cloud_etcds.yaml index 49b507783..d0c7d23cc 100644 --- a/charts/druid/charts/crds/templates/crd-druid.gardener.cloud_etcds.yaml +++ b/charts/druid/charts/crds/templates/crd-druid.gardener.cloud_etcds.yaml @@ -597,6 +597,12 @@ spec: serverPort: format: int32 type: integer + snapshotCount: + description: |- + SnapshotCount defines the number of applied Raft entries to hold in-memory before compaction. + More info: https://etcd.io/docs/v3.4/op-guide/maintenance/#raft-log-retention + format: int64 + type: integer type: object labels: additionalProperties: diff --git a/config/crd/bases/crd-druid.gardener.cloud_etcds.yaml b/config/crd/bases/crd-druid.gardener.cloud_etcds.yaml index 49b507783..d0c7d23cc 100644 --- a/config/crd/bases/crd-druid.gardener.cloud_etcds.yaml +++ b/config/crd/bases/crd-druid.gardener.cloud_etcds.yaml @@ -597,6 +597,12 @@ spec: serverPort: format: int32 type: integer + snapshotCount: + description: |- + SnapshotCount defines the number of applied Raft entries to hold in-memory before compaction. + More info: https://etcd.io/docs/v3.4/op-guide/maintenance/#raft-log-retention + format: int64 + type: integer type: object labels: additionalProperties: diff --git a/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md b/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md new file mode 100644 index 000000000..83ec946c9 --- /dev/null +++ b/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md @@ -0,0 +1,131 @@ +# Recovery from Permanent Quorum Loss in an Etcd Cluster + +## Quorum loss in Etcd Cluster +[Quorum loss](https://etcd.io/docs/v3.4/op-guide/recovery/) means when the majority of Etcd pods (greater than or equal to n/2 + 1) are down simultaneously for some reason. + +There are two types of quorum loss that can happen to an [Etcd multinode cluster](../proposals/01-multi-node-etcd-clusters.md): + +1. **Transient quorum loss** - A quorum loss is called transient when the majority of Etcd pods are down simultaneously for some time. The pods may be down due to network unavailability, high resource usages, etc. When the pods come back after some time, they can re-join the cluster and quorum is recovered automatically without any manual intervention. There should not be a permanent failure for the majority of etcd pods due to hardware failure or disk corruption. + +2. **Permanent quorum loss** - A quorum loss is called permanent when the majority of Etcd cluster members experience permanent failure, whether due to hardware failure or disk corruption, etc. In that case, the etcd cluster is not going to recover automatically from the quorum loss. A human operator will now need to intervene and execute the following steps to recover the multi-node Etcd cluster. + +If permanent quorum loss occurs to a multinode Etcd cluster, the operator needs to note down the PVCs, configmaps, statefulsets, CRs, etc. related to that Etcd cluster and work on those resources only. The following steps guide a human operator to recover from permanent quorum loss of an etcd cluster. We assume the name of the Etcd CR for the Etcd cluster is `etcd-main`. + +**Etcd cluster in shoot control plane of gardener deployment:** +There are two [Etcd clusters](../proposals/01-multi-node-etcd-clusters.md) running in the shoot control plane. One is named `etcd-events` and another is named `etcd-main`. The operator needs to take care of permanent quorum loss to a specific cluster. If permanent quorum loss occurs to `etcd-events` cluster, the operator needs to note down the PVCs, configmaps, statefulsets, CRs, etc. related to the `etcd-events` cluster and work on those resources only. + +:warning: **Note:** Please note that manually restoring etcd can result in data loss. This guide is the last resort to bring an Etcd cluster up and running again. + +If etcd-druid and etcd-backup-restore is being used with gardener, then: + +Target the control plane of affected shoot cluster via `kubectl`. Alternatively, you can use [gardenctl](https://github.com/gardener/gardenctl-v2) to target the control plane of the affected shoot cluster. You can get the details to target the control plane from the Access tile in the shoot cluster details page on the Gardener dashboard. Ensure that you are targeting the correct namespace. + +1. Add the following annotations to the `Etcd` resource `etcd-main`: + 1. `kubectl annotate etcd etcd-main druid.gardener.cloud/suspend-etcd-spec-reconcile=` + + 2. `kubectl annotate etcd etcd-main druid.gardener.cloud/disable-resource-protection=` + +2. Note down the configmap name that is attached to the `etcd-main` statefulset. If you describe the statefulset with `kubectl describe sts etcd-main`, look for the lines similar to following lines to identify attached configmap name. It will be needed at later stages: + + ``` + Volumes: + etcd-config-file: + Type: ConfigMap (a volume populated by a ConfigMap) + Name: etcd-main-config-4785b0 + Optional: false + ``` + + Alternatively, the related configmap name can be obtained by executing following command as well: + + `kubectl get sts etcd-main -o jsonpath='{.spec.template.spec.volumes[?(@.name=="etcd-config-file")].configMap.name}'` + +3. Scale down the `etcd-main` statefulset replicas to `0`: + + `kubectl scale sts etcd-main --replicas=0` + +4. The PVCs will look like the following on listing them with the command `kubectl get pvc`: + + ``` + main-etcd-etcd-main-0 Bound pv-shoot--garden--aws-ha-dcb51848-49fa-4501-b2f2-f8d8f1fad111 80Gi RWO gardener.cloud-fast 13d + main-etcd-etcd-main-1 Bound pv-shoot--garden--aws-ha-b4751b28-c06e-41b7-b08c-6486e03090dd 80Gi RWO gardener.cloud-fast 13d + main-etcd-etcd-main-2 Bound pv-shoot--garden--aws-ha-ff17323b-d62e-4d5e-a742-9de823621490 80Gi RWO gardener.cloud-fast 13d + ``` + Delete all PVCs that are attached to `etcd-main` cluster. + + `kubectl delete pvc -l instance=etcd-main` + +5. Check the etcd's member leases. There should be leases starting with `etcd-main` as many as `etcd-main` replicas. + One of those leases will have holder identity as `:Leader` and rest of etcd member leases have holder identities as `:Member`. + Please ignore the snapshot leases, i.e., those leases which have the suffix `snap`. + + etcd-main member leases: + ``` + NAME HOLDER AGE + etcd-main-0 4c37667312a3912b:Member 1m + etcd-main-1 75a9b74cfd3077cc:Member 1m + etcd-main-2 c62ee6af755e890d:Leader 1m + ``` + + Delete all `etcd-main` member leases. + +6. Edit the `etcd-main` cluster's configmap (ex: `etcd-main-config-4785b0`) as follows: + + Find the `initial-cluster` field in the configmap. It should look similar to the following: + ``` + # Initial cluster + initial-cluster: etcd-main-0=https://etcd-main-0.etcd-main-peer.default.svc:2380,etcd-main-1=https://etcd-main-1.etcd-main-peer.default.svc:2380,etcd-main-2=https://etcd-main-2.etcd-main-peer.default.svc:2380 + ``` + + Change the `initial-cluster` field to have only one member (`etcd-main-0`) in the string. It should now look like this: + + ``` + # Initial cluster + initial-cluster: etcd-main-0=https://etcd-main-0.etcd-main-peer.default.svc:2380 + ``` + +7. Scale up the `etcd-main` statefulset replicas to `1`: + + `kubectl scale sts etcd-main --replicas=1` + +8. Wait for the single-member etcd cluster to be completely ready. + + `kubectl get pods etcd-main-0` will give the following output when ready: + ``` + NAME READY STATUS RESTARTS AGE + etcd-main-0 2/2 Running 0 1m + ``` + +9. Remove the following annotations from the `Etcd` resource `etcd-main`: + + 1. `kubectl annotate etcd etcd-main druid.gardener.cloud/suspend-etcd-spec-reconcile-` + + 2. `kubectl annotate etcd etcd-main druid.gardener.cloud/disable-resource-protection-` + +10. Finally, add the following annotation to the `Etcd` resource `etcd-main`: + + `kubectl annotate etcd etcd-main gardener.cloud/operation='reconcile'` + +11. Verify that the etcd cluster is formed correctly. + + All the `etcd-main` pods will have outputs similar to following: + ``` + NAME READY STATUS RESTARTS AGE + etcd-main-0 2/2 Running 0 5m + etcd-main-1 2/2 Running 0 1m + etcd-main-2 2/2 Running 0 1m + ``` + Additionally, check if the Etcd CR is ready with `kubectl get etcd etcd-main`: + ``` + NAME READY AGE + etcd-main true 13d + ``` + + Additionally, check the leases for 30 seconds at least. There should be leases starting with `etcd-main` as many as `etcd-main` replicas. One of those leases will have holder identity as `:Leader` and rest of those leases have holder identities as `:Member`. The `AGE` of those leases can also be inspected to identify if those leases were updated in conjunction with the restart of the Etcd cluster: Example: + + ``` + NAME HOLDER AGE + etcd-main-0 4c37667312a3912b:Member 1m + etcd-main-1 75a9b74cfd3077cc:Member 1m + etcd-main-2 c62ee6af755e890d:Leader 1m + ``` + diff --git a/docs/proposals/03-scaling-up-an-etcd-cluster.md b/docs/proposals/03-scaling-up-an-etcd-cluster.md index 51af42f94..3de353360 100644 --- a/docs/proposals/03-scaling-up-an-etcd-cluster.md +++ b/docs/proposals/03-scaling-up-an-etcd-cluster.md @@ -24,7 +24,7 @@ Now, it is detected whether peer URL was TLS enabled or not for single node etcd - If peer URL was not TLS enabled then etcd-druid has to intervene and make sure peer URL should be TLS enabled first for the single node before marking the cluster for scale-up. ## Action taken by etcd-druid to enable the peerURL TLS -1. Etcd-druid will update the `etcd-bootstrap` config-map with new config like initial-cluster,initial-advertise-peer-urls etc. Backup-restore will detect this change and update the member lease annotation to `member.etcd.gardener.cloud/tls-enabled: "true"`. +1. Etcd-druid will update the `{etcd.Name}-config` config-map with new config like initial-cluster,initial-advertise-peer-urls etc. Backup-restore will detect this change and update the member lease annotation to `member.etcd.gardener.cloud/tls-enabled: "true"`. 2. In case the peer URL TLS has been changed to `enabled`: Etcd-druid will add tasks to the deployment flow: - Check if peer TLS has been enabled for existing StatefulSet pods, by checking the member leases for the annotation `member.etcd.gardener.cloud/tls-enabled`. - If peer TLS enablement is pending for any of the members, then check and patch the StatefulSet with the peer TLS volume mounts, if not already patched. This will cause a rolling update of the existing StatefulSet pods, which allows etcd-backup-restore to update the member peer URL in the etcd cluster. diff --git a/internal/component/configmap/configmap_test.go b/internal/component/configmap/configmap_test.go index 3fa558a65..2f1f9e6eb 100644 --- a/internal/component/configmap/configmap_test.go +++ b/internal/component/configmap/configmap_test.go @@ -7,7 +7,6 @@ package configmap import ( "context" "fmt" - "strconv" "testing" druidv1alpha1 "github.com/gardener/etcd-druid/api/v1alpha1" @@ -344,7 +343,7 @@ func matchConfigMap(g *WithT, etcd *druidv1alpha1.Etcd, actualConfigMap corev1.C "name": Equal(fmt.Sprintf("etcd-%s", etcd.UID[:6])), "data-dir": Equal(fmt.Sprintf("%s/new.etcd", common.VolumeMountPathEtcdData)), "metrics": Equal(string(druidv1alpha1.Basic)), - "snapshot-count": Equal(int64(75000)), + "snapshot-count": Equal(ptr.Deref(etcd.Spec.Etcd.SnapshotCount, defaultSnapshotCount)), "enable-v2": Equal(false), "quota-backend-bytes": Equal(etcd.Spec.Etcd.Quota.Value()), "initial-cluster-token": Equal("etcd-cluster"), @@ -360,7 +359,7 @@ func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actu if etcd.Spec.Etcd.ClientUrlTLS != nil { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", ptr.Deref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), - "advertise-client-urls": Equal(fmt.Sprintf("https@%s@%s@%d", druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta), etcd.Namespace, ptr.Deref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), + "advertise-client-urls": Equal(fmt.Sprintf("https://%s.%s:%d", druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta), etcd.Namespace, ptr.Deref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), "client-transport-security": MatchKeys(IgnoreExtras, Keys{ "cert-file": Equal("/var/etcd/ssl/server/tls.crt"), "key-file": Equal("/var/etcd/ssl/server/tls.key"), @@ -389,12 +388,12 @@ func matchPeerTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actual "auto-tls": Equal(false), }), "listen-peer-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), - "initial-advertise-peer-urls": Equal(fmt.Sprintf("https@%s@%s@%s", peerSvcName, etcd.Namespace, strconv.Itoa(int(ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))))), + "initial-advertise-peer-urls": Equal(fmt.Sprintf("https://%s.%s:%d", peerSvcName, etcd.Namespace, ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), })) } else { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ "listen-peer-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), - "initial-advertise-peer-urls": Equal(fmt.Sprintf("http@%s@%s@%s", peerSvcName, etcd.Namespace, strconv.Itoa(int(ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))))), + "initial-advertise-peer-urls": Equal(fmt.Sprintf("http://%s.%s:%d", peerSvcName, etcd.Namespace, ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), })) g.Expect(actualETCDConfig).ToNot(HaveKey("peer-transport-security")) } diff --git a/internal/component/configmap/etcdconfig.go b/internal/component/configmap/etcdconfig.go index de9c8409a..67f57bc01 100644 --- a/internal/component/configmap/etcdconfig.go +++ b/internal/component/configmap/etcdconfig.go @@ -22,27 +22,22 @@ const ( defaultInitialClusterToken = "etcd-cluster" defaultInitialClusterState = "new" // For more information refer to https://etcd.io/docs/v3.4/op-guide/maintenance/#raft-log-retention - // TODO: Ideally this should be made configurable via Etcd resource as this has a direct impact on the memory requirements for etcd container. - // which in turn is influenced by the size of objects that are getting stored in etcd. - defaultSnapshotCount = 75000 + defaultSnapshotCount = int64(75000) ) var ( defaultDataDir = fmt.Sprintf("%s/new.etcd", common.VolumeMountPathEtcdData) ) -type tlsTarget string +type advPeerUrls map[string][]string -const ( - clientTLS tlsTarget = "client" - peerTLS tlsTarget = "peer" -) +type advClientUrls map[string]string type etcdConfig struct { Name string `yaml:"name"` DataDir string `yaml:"data-dir"` Metrics druidv1alpha1.MetricsLevel `yaml:"metrics"` - SnapshotCount int `yaml:"snapshot-count"` + SnapshotCount int64 `yaml:"snapshot-count"` EnableV2 bool `yaml:"enable-v2"` QuotaBackendBytes int64 `yaml:"quota-backend-bytes"` InitialClusterToken string `yaml:"initial-cluster-token"` @@ -52,8 +47,8 @@ type etcdConfig struct { AutoCompactionRetention string `yaml:"auto-compaction-retention"` ListenPeerUrls string `yaml:"listen-peer-urls"` ListenClientUrls string `yaml:"listen-client-urls"` - AdvertisePeerUrls string `yaml:"initial-advertise-peer-urls"` - AdvertiseClientUrls string `yaml:"advertise-client-urls"` + AdvertisePeerUrls advPeerUrls `yaml:"initial-advertise-peer-urls"` + AdvertiseClientUrls advClientUrls `yaml:"advertise-client-urls"` ClientSecurity securityConfig `yaml:"client-transport-security,omitempty"` PeerSecurity securityConfig `yaml:"peer-transport-security,omitempty"` } @@ -74,7 +69,7 @@ func createEtcdConfig(etcd *druidv1alpha1.Etcd) *etcdConfig { Name: fmt.Sprintf("etcd-%s", etcd.UID[:6]), DataDir: defaultDataDir, Metrics: ptr.Deref(etcd.Spec.Etcd.Metrics, druidv1alpha1.Basic), - SnapshotCount: defaultSnapshotCount, + SnapshotCount: getSnapshotCount(etcd), EnableV2: false, QuotaBackendBytes: getDBQuotaBytes(etcd), InitialClusterToken: defaultInitialClusterToken, @@ -84,8 +79,8 @@ func createEtcdConfig(etcd *druidv1alpha1.Etcd) *etcdConfig { AutoCompactionRetention: ptr.Deref(etcd.Spec.Common.AutoCompactionRetention, defaultAutoCompactionRetention), ListenPeerUrls: fmt.Sprintf("%s://0.0.0.0:%d", peerScheme, ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer)), ListenClientUrls: fmt.Sprintf("%s://0.0.0.0:%d", clientScheme, ptr.Deref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient)), - AdvertisePeerUrls: fmt.Sprintf("%s@%s@%s@%d", peerScheme, peerSvcName, etcd.Namespace, ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer)), - AdvertiseClientUrls: fmt.Sprintf("%s@%s@%s@%d", clientScheme, peerSvcName, etcd.Namespace, ptr.Deref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient)), + AdvertisePeerUrls: getAdvertisePeerUrlsMap(etcd, peerScheme, peerSvcName), + AdvertiseClientUrls: getAdvertiseClientUrlMap(etcd, clientScheme, peerSvcName), } if peerSecurityConfig != nil { cfg.PeerSecurity = *peerSecurityConfig @@ -97,6 +92,14 @@ func createEtcdConfig(etcd *druidv1alpha1.Etcd) *etcdConfig { return cfg } +func getSnapshotCount(etcd *druidv1alpha1.Etcd) int64 { + snapshotCount := defaultSnapshotCount + if etcd.Spec.Etcd.SnapshotCount != nil { + snapshotCount = *etcd.Spec.Etcd.SnapshotCount + } + return snapshotCount +} + func getDBQuotaBytes(etcd *druidv1alpha1.Etcd) int64 { dbQuotaBytes := defaultDBQuotaBytes if etcd.Spec.Etcd.Quota != nil { @@ -129,3 +132,21 @@ func prepareInitialCluster(etcd *druidv1alpha1.Etcd, peerScheme string) string { } return strings.Trim(builder.String(), ",") } + +func getAdvertisePeerUrlsMap(etcd *druidv1alpha1.Etcd, peerScheme string, peerSvcName string) advPeerUrls { + advPeerUrlsMap := make(map[string][]string) + for i := 0; i < int(etcd.Spec.Replicas); i++ { + podName := druidv1alpha1.GetOrdinalPodName(etcd.ObjectMeta, i) + advPeerUrlsMap[podName] = []string{fmt.Sprintf("%s://%s.%s.%s.svc:%d", peerScheme, podName, peerSvcName, etcd.Namespace, ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))} + } + return advPeerUrlsMap +} + +func getAdvertiseClientUrlMap(etcd *druidv1alpha1.Etcd, clientScheme string, peerSvcName string) advClientUrls { + advClientUrlMap := make(map[string]string) + for i := 0; i < int(etcd.Spec.Replicas); i++ { + podName := druidv1alpha1.GetOrdinalPodName(etcd.ObjectMeta, i) + advClientUrlMap[podName] = fmt.Sprintf("%s://%s.%s.%s.svc:%d", clientScheme, podName, peerSvcName, etcd.Namespace, ptr.Deref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient)) + } + return advClientUrlMap +} diff --git a/test/e2e/etcd_backup_test.go b/test/e2e/etcd_backup_test.go index a4a2727bf..9dd840c2e 100644 --- a/test/e2e/etcd_backup_test.go +++ b/test/e2e/etcd_backup_test.go @@ -247,7 +247,7 @@ func checkEtcdReady(ctx context.Context, cl client.Client, logger logr.Logger, e logger.Info("Checking configmap") cm := &corev1.ConfigMap{} - ExpectWithOffset(2, cl.Get(ctx, client.ObjectKey{Name: "etcd-bootstrap-" + string(etcd.UID[:6]), Namespace: etcd.Namespace}, cm)).To(Succeed()) + ExpectWithOffset(2, cl.Get(ctx, client.ObjectKey{Name: etcd.Name + "-config-" + string(etcd.UID[:6]), Namespace: etcd.Namespace}, cm)).To(Succeed()) logger.Info("Checking client service") svc := &corev1.Service{} @@ -280,7 +280,7 @@ func deleteAndCheckEtcd(ctx context.Context, cl client.Client, logger logr.Logge ExpectWithOffset(1, cl.Get( ctx, - client.ObjectKey{Name: "etcd-bootstrap-" + string(etcd.UID[:6]), Namespace: etcd.Namespace}, + client.ObjectKey{Name: etcd.Name + "-config-" + string(etcd.UID[:6]), Namespace: etcd.Namespace}, &corev1.ConfigMap{}, ), ).Should(matchers.BeNotFoundError()) diff --git a/test/e2e/utils.go b/test/e2e/utils.go index f24aafb02..56841b207 100644 --- a/test/e2e/utils.go +++ b/test/e2e/utils.go @@ -105,8 +105,9 @@ var ( "memory": resource.MustParse("256Mi"), }, } - etcdClientPort = int32(2379) - etcdServerPort = int32(2380) + etcdClientPort = int32(2379) + etcdServerPort = int32(2380) + etcdSnapshotCount = int64(75000) backupPort = int32(8080) backupFullSnapshotSchedule = "0 */1 * * *" @@ -182,6 +183,7 @@ func getDefaultEtcd(name, namespace, container, prefix string, provider TestProv Resources: &etcdResources, ClientPort: &etcdClientPort, ServerPort: &etcdServerPort, + SnapshotCount: &etcdSnapshotCount, ClientUrlTLS: &etcdTLS, } diff --git a/test/it/setup/setup.go b/test/it/setup/setup.go index b190b6802..b94d94d3b 100644 --- a/test/it/setup/setup.go +++ b/test/it/setup/setup.go @@ -20,7 +20,7 @@ import ( k8sruntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" - "k8s.io/utils/pointer" + "k8s.io/utils/ptr" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/envtest" @@ -168,7 +168,7 @@ func (t *itTestEnv) startTestEnvironment(crdDirectoryPaths []string) error { CRDDirectoryPaths: crdDirectoryPaths, } if useExistingK8SCluster() { - testEnv.UseExistingCluster = pointer.Bool(true) + testEnv.UseExistingCluster = ptr.To(true) } cfg, err := testEnv.Start() diff --git a/test/utils/etcd.go b/test/utils/etcd.go index fbfd2a752..b2c2658f6 100644 --- a/test/utils/etcd.go +++ b/test/utils/etcd.go @@ -40,6 +40,7 @@ var ( deltaSnapShotMemLimit = resource.MustParse("100Mi") autoCompactionMode = druidv1alpha1.Periodic autoCompactionRetention = "2m" + snapshotCount = int64(75000) quota = resource.MustParse("8Gi") localProvider = druidv1alpha1.StorageProvider("Local") prefix = "/tmp" @@ -387,6 +388,7 @@ func getDefaultEtcd(name, namespace string) *druidv1alpha1.Etcd { Backup: getBackupSpec(), Etcd: druidv1alpha1.EtcdConfig{ Quota: "a, + SnapshotCount: &snapshotCount, Metrics: &metricsBasic, Image: &imageEtcd, DefragmentationSchedule: &defragSchedule, From 3d3205e68b5daec437231145bbbf8c73b53611fe Mon Sep 17 00:00:00 2001 From: Anvesh Reddy Pinnapureddy Date: Tue, 15 Oct 2024 11:01:55 +0530 Subject: [PATCH 2/8] fix tests --- .../component/configmap/configmap_test.go | 35 +++++++++++++------ 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/internal/component/configmap/configmap_test.go b/internal/component/configmap/configmap_test.go index 2f1f9e6eb..ba58f510c 100644 --- a/internal/component/configmap/configmap_test.go +++ b/internal/component/configmap/configmap_test.go @@ -305,12 +305,6 @@ func newConfigMap(g *WithT, etcd *druidv1alpha1.Etcd) *corev1.ConfigMap { return cm } -func ensureConfigMapExists(g *WithT, cl client.WithWatch, etcd *druidv1alpha1.Etcd) { - cm, err := getLatestConfigMap(cl, etcd) - g.Expect(err).ToNot(HaveOccurred()) - g.Expect(cm).ToNot(BeNil()) -} - func getLatestConfigMap(cl client.Client, etcd *druidv1alpha1.Etcd) (*corev1.ConfigMap, error) { cm := &corev1.ConfigMap{} err := cl.Get(context.Background(), client.ObjectKey{Name: druidv1alpha1.GetConfigMapName(etcd.ObjectMeta), Namespace: etcd.Namespace}, cm) @@ -359,7 +353,7 @@ func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actu if etcd.Spec.Etcd.ClientUrlTLS != nil { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", ptr.Deref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), - "advertise-client-urls": Equal(fmt.Sprintf("https://%s.%s:%d", druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta), etcd.Namespace, ptr.Deref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), + "advertise-client-urls": Equal(getAdvertiseInterface(etcd, "https")), "client-transport-security": MatchKeys(IgnoreExtras, Keys{ "cert-file": Equal("/var/etcd/ssl/server/tls.crt"), "key-file": Equal("/var/etcd/ssl/server/tls.key"), @@ -376,8 +370,29 @@ func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actu } } +func getAdvertiseInterface(etcd *druidv1alpha1.Etcd, scheme string) map[string]interface{} { + advertiseClientUrls := getAdvertiseClientUrlMap(etcd, scheme, druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta)) + advertiseClientUrlsInterface := make(map[string]interface{}, len(advertiseClientUrls)) + for k, v := range advertiseClientUrls { + advertiseClientUrlsInterface[k] = v + } + return advertiseClientUrlsInterface +} + +func getAdvertisePeerInterface(etcd *druidv1alpha1.Etcd, scheme string) map[string]interface{} { + advertisePeerUrls := getAdvertisePeerUrlsMap(etcd, scheme, druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta)) + advertisePeerUrlsInterface := make(map[string]interface{}, len(advertisePeerUrls)) + for k, v := range advertisePeerUrls { + urlsInterface := make([]interface{}, len(v)) + for i, url := range v { + urlsInterface[i] = url + } + advertisePeerUrlsInterface[k] = urlsInterface + } + return advertisePeerUrlsInterface +} + func matchPeerTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actualETCDConfig map[string]interface{}) { - peerSvcName := druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta) if etcd.Spec.Etcd.PeerUrlTLS != nil { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ "peer-transport-security": MatchKeys(IgnoreExtras, Keys{ @@ -388,12 +403,12 @@ func matchPeerTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actual "auto-tls": Equal(false), }), "listen-peer-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), - "initial-advertise-peer-urls": Equal(fmt.Sprintf("https://%s.%s:%d", peerSvcName, etcd.Namespace, ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), + "initial-advertise-peer-urls": Equal(getAdvertisePeerInterface(etcd, "https")), })) } else { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ "listen-peer-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), - "initial-advertise-peer-urls": Equal(fmt.Sprintf("http://%s.%s:%d", peerSvcName, etcd.Namespace, ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), + "initial-advertise-peer-urls": Equal(getAdvertisePeerInterface(etcd, "http")), })) g.Expect(actualETCDConfig).ToNot(HaveKey("peer-transport-security")) } From 810fc19c10e6ee904d8a7e31b7b099bd2bd641e2 Mon Sep 17 00:00:00 2001 From: Anvesh Reddy Pinnapureddy Date: Tue, 22 Oct 2024 13:06:38 +0530 Subject: [PATCH 3/8] Use list for advertise-client-urls as well --- .../component/configmap/configmap_test.go | 33 ++++++---------- internal/component/configmap/etcdconfig.go | 38 +++++++++---------- 2 files changed, 31 insertions(+), 40 deletions(-) diff --git a/internal/component/configmap/configmap_test.go b/internal/component/configmap/configmap_test.go index ba58f510c..15cc25a97 100644 --- a/internal/component/configmap/configmap_test.go +++ b/internal/component/configmap/configmap_test.go @@ -353,7 +353,7 @@ func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actu if etcd.Spec.Etcd.ClientUrlTLS != nil { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", ptr.Deref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), - "advertise-client-urls": Equal(getAdvertiseInterface(etcd, "https")), + "advertise-client-urls": Equal(getAdvertiseURLsInterface(etcd, commTypeClient, "https")), "client-transport-security": MatchKeys(IgnoreExtras, Keys{ "cert-file": Equal("/var/etcd/ssl/server/tls.crt"), "key-file": Equal("/var/etcd/ssl/server/tls.key"), @@ -370,26 +370,17 @@ func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actu } } -func getAdvertiseInterface(etcd *druidv1alpha1.Etcd, scheme string) map[string]interface{} { - advertiseClientUrls := getAdvertiseClientUrlMap(etcd, scheme, druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta)) - advertiseClientUrlsInterface := make(map[string]interface{}, len(advertiseClientUrls)) - for k, v := range advertiseClientUrls { - advertiseClientUrlsInterface[k] = v - } - return advertiseClientUrlsInterface -} - -func getAdvertisePeerInterface(etcd *druidv1alpha1.Etcd, scheme string) map[string]interface{} { - advertisePeerUrls := getAdvertisePeerUrlsMap(etcd, scheme, druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta)) - advertisePeerUrlsInterface := make(map[string]interface{}, len(advertisePeerUrls)) - for k, v := range advertisePeerUrls { - urlsInterface := make([]interface{}, len(v)) - for i, url := range v { - urlsInterface[i] = url +func getAdvertiseURLsInterface(etcd *druidv1alpha1.Etcd, commType, scheme string) map[string]interface{} { + advertiseUrlsMap := getAdvertiseUrlsMap(etcd, commType, scheme, druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta)) + advertiseUrlsInterface := make(map[string]interface{}, len(advertiseUrlsMap)) + for podName, urlList := range advertiseUrlsMap { + urlsListInterface := make([]interface{}, len(urlList)) + for i, url := range urlList { + urlsListInterface[i] = url } - advertisePeerUrlsInterface[k] = urlsInterface + advertiseUrlsInterface[podName] = urlsListInterface } - return advertisePeerUrlsInterface + return advertiseUrlsInterface } func matchPeerTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actualETCDConfig map[string]interface{}) { @@ -403,12 +394,12 @@ func matchPeerTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actual "auto-tls": Equal(false), }), "listen-peer-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), - "initial-advertise-peer-urls": Equal(getAdvertisePeerInterface(etcd, "https")), + "initial-advertise-peer-urls": Equal(getAdvertiseURLsInterface(etcd, commTypePeer, "https")), })) } else { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ "listen-peer-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), - "initial-advertise-peer-urls": Equal(getAdvertisePeerInterface(etcd, "http")), + "initial-advertise-peer-urls": Equal(getAdvertiseURLsInterface(etcd, commTypePeer, "http")), })) g.Expect(actualETCDConfig).ToNot(HaveKey("peer-transport-security")) } diff --git a/internal/component/configmap/etcdconfig.go b/internal/component/configmap/etcdconfig.go index 67f57bc01..5133d66a4 100644 --- a/internal/component/configmap/etcdconfig.go +++ b/internal/component/configmap/etcdconfig.go @@ -23,15 +23,15 @@ const ( defaultInitialClusterState = "new" // For more information refer to https://etcd.io/docs/v3.4/op-guide/maintenance/#raft-log-retention defaultSnapshotCount = int64(75000) + commTypePeer = "peer" // communication type + commTypeClient = "client" ) var ( defaultDataDir = fmt.Sprintf("%s/new.etcd", common.VolumeMountPathEtcdData) ) -type advPeerUrls map[string][]string - -type advClientUrls map[string]string +type advertiseURLs map[string][]string type etcdConfig struct { Name string `yaml:"name"` @@ -47,8 +47,8 @@ type etcdConfig struct { AutoCompactionRetention string `yaml:"auto-compaction-retention"` ListenPeerUrls string `yaml:"listen-peer-urls"` ListenClientUrls string `yaml:"listen-client-urls"` - AdvertisePeerUrls advPeerUrls `yaml:"initial-advertise-peer-urls"` - AdvertiseClientUrls advClientUrls `yaml:"advertise-client-urls"` + AdvertisePeerUrls advertiseURLs `yaml:"initial-advertise-peer-urls"` + AdvertiseClientUrls advertiseURLs `yaml:"advertise-client-urls"` ClientSecurity securityConfig `yaml:"client-transport-security,omitempty"` PeerSecurity securityConfig `yaml:"peer-transport-security,omitempty"` } @@ -79,8 +79,8 @@ func createEtcdConfig(etcd *druidv1alpha1.Etcd) *etcdConfig { AutoCompactionRetention: ptr.Deref(etcd.Spec.Common.AutoCompactionRetention, defaultAutoCompactionRetention), ListenPeerUrls: fmt.Sprintf("%s://0.0.0.0:%d", peerScheme, ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer)), ListenClientUrls: fmt.Sprintf("%s://0.0.0.0:%d", clientScheme, ptr.Deref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient)), - AdvertisePeerUrls: getAdvertisePeerUrlsMap(etcd, peerScheme, peerSvcName), - AdvertiseClientUrls: getAdvertiseClientUrlMap(etcd, clientScheme, peerSvcName), + AdvertisePeerUrls: getAdvertiseUrlsMap(etcd, commTypePeer, peerScheme, peerSvcName), + AdvertiseClientUrls: getAdvertiseUrlsMap(etcd, commTypeClient, clientScheme, peerSvcName), } if peerSecurityConfig != nil { cfg.PeerSecurity = *peerSecurityConfig @@ -133,20 +133,20 @@ func prepareInitialCluster(etcd *druidv1alpha1.Etcd, peerScheme string) string { return strings.Trim(builder.String(), ",") } -func getAdvertisePeerUrlsMap(etcd *druidv1alpha1.Etcd, peerScheme string, peerSvcName string) advPeerUrls { - advPeerUrlsMap := make(map[string][]string) - for i := 0; i < int(etcd.Spec.Replicas); i++ { - podName := druidv1alpha1.GetOrdinalPodName(etcd.ObjectMeta, i) - advPeerUrlsMap[podName] = []string{fmt.Sprintf("%s://%s.%s.%s.svc:%d", peerScheme, podName, peerSvcName, etcd.Namespace, ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))} +func getAdvertiseUrlsMap(etcd *druidv1alpha1.Etcd, commType, scheme, peerSvcName string) advertiseURLs { + var port int32 + switch commType { + case commTypePeer: + port = ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer) + case commTypeClient: + port = ptr.Deref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient) + default: + return nil } - return advPeerUrlsMap -} - -func getAdvertiseClientUrlMap(etcd *druidv1alpha1.Etcd, clientScheme string, peerSvcName string) advClientUrls { - advClientUrlMap := make(map[string]string) + advUrlsMap := make(map[string][]string) for i := 0; i < int(etcd.Spec.Replicas); i++ { podName := druidv1alpha1.GetOrdinalPodName(etcd.ObjectMeta, i) - advClientUrlMap[podName] = fmt.Sprintf("%s://%s.%s.%s.svc:%d", clientScheme, podName, peerSvcName, etcd.Namespace, ptr.Deref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient)) + advUrlsMap[podName] = []string{fmt.Sprintf("%s://%s.%s.%s.svc:%d", scheme, podName, peerSvcName, etcd.Namespace, port)} } - return advClientUrlMap + return advUrlsMap } From 5fc627c25d649deb731a916e2ccaeeb816756de8 Mon Sep 17 00:00:00 2001 From: Anvesh Reddy Pinnapureddy Date: Thu, 24 Oct 2024 22:37:51 +0530 Subject: [PATCH 4/8] Address review comments by @ishan16696 --- ...m-permanent-quorum-loss-in-etcd-cluster.md | 131 ------------------ .../component/configmap/configmap_test.go | 2 +- internal/component/configmap/etcdconfig.go | 16 +-- 3 files changed, 8 insertions(+), 141 deletions(-) delete mode 100644 docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md diff --git a/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md b/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md deleted file mode 100644 index 83ec946c9..000000000 --- a/docs/operations/recovery-from-permanent-quorum-loss-in-etcd-cluster.md +++ /dev/null @@ -1,131 +0,0 @@ -# Recovery from Permanent Quorum Loss in an Etcd Cluster - -## Quorum loss in Etcd Cluster -[Quorum loss](https://etcd.io/docs/v3.4/op-guide/recovery/) means when the majority of Etcd pods (greater than or equal to n/2 + 1) are down simultaneously for some reason. - -There are two types of quorum loss that can happen to an [Etcd multinode cluster](../proposals/01-multi-node-etcd-clusters.md): - -1. **Transient quorum loss** - A quorum loss is called transient when the majority of Etcd pods are down simultaneously for some time. The pods may be down due to network unavailability, high resource usages, etc. When the pods come back after some time, they can re-join the cluster and quorum is recovered automatically without any manual intervention. There should not be a permanent failure for the majority of etcd pods due to hardware failure or disk corruption. - -2. **Permanent quorum loss** - A quorum loss is called permanent when the majority of Etcd cluster members experience permanent failure, whether due to hardware failure or disk corruption, etc. In that case, the etcd cluster is not going to recover automatically from the quorum loss. A human operator will now need to intervene and execute the following steps to recover the multi-node Etcd cluster. - -If permanent quorum loss occurs to a multinode Etcd cluster, the operator needs to note down the PVCs, configmaps, statefulsets, CRs, etc. related to that Etcd cluster and work on those resources only. The following steps guide a human operator to recover from permanent quorum loss of an etcd cluster. We assume the name of the Etcd CR for the Etcd cluster is `etcd-main`. - -**Etcd cluster in shoot control plane of gardener deployment:** -There are two [Etcd clusters](../proposals/01-multi-node-etcd-clusters.md) running in the shoot control plane. One is named `etcd-events` and another is named `etcd-main`. The operator needs to take care of permanent quorum loss to a specific cluster. If permanent quorum loss occurs to `etcd-events` cluster, the operator needs to note down the PVCs, configmaps, statefulsets, CRs, etc. related to the `etcd-events` cluster and work on those resources only. - -:warning: **Note:** Please note that manually restoring etcd can result in data loss. This guide is the last resort to bring an Etcd cluster up and running again. - -If etcd-druid and etcd-backup-restore is being used with gardener, then: - -Target the control plane of affected shoot cluster via `kubectl`. Alternatively, you can use [gardenctl](https://github.com/gardener/gardenctl-v2) to target the control plane of the affected shoot cluster. You can get the details to target the control plane from the Access tile in the shoot cluster details page on the Gardener dashboard. Ensure that you are targeting the correct namespace. - -1. Add the following annotations to the `Etcd` resource `etcd-main`: - 1. `kubectl annotate etcd etcd-main druid.gardener.cloud/suspend-etcd-spec-reconcile=` - - 2. `kubectl annotate etcd etcd-main druid.gardener.cloud/disable-resource-protection=` - -2. Note down the configmap name that is attached to the `etcd-main` statefulset. If you describe the statefulset with `kubectl describe sts etcd-main`, look for the lines similar to following lines to identify attached configmap name. It will be needed at later stages: - - ``` - Volumes: - etcd-config-file: - Type: ConfigMap (a volume populated by a ConfigMap) - Name: etcd-main-config-4785b0 - Optional: false - ``` - - Alternatively, the related configmap name can be obtained by executing following command as well: - - `kubectl get sts etcd-main -o jsonpath='{.spec.template.spec.volumes[?(@.name=="etcd-config-file")].configMap.name}'` - -3. Scale down the `etcd-main` statefulset replicas to `0`: - - `kubectl scale sts etcd-main --replicas=0` - -4. The PVCs will look like the following on listing them with the command `kubectl get pvc`: - - ``` - main-etcd-etcd-main-0 Bound pv-shoot--garden--aws-ha-dcb51848-49fa-4501-b2f2-f8d8f1fad111 80Gi RWO gardener.cloud-fast 13d - main-etcd-etcd-main-1 Bound pv-shoot--garden--aws-ha-b4751b28-c06e-41b7-b08c-6486e03090dd 80Gi RWO gardener.cloud-fast 13d - main-etcd-etcd-main-2 Bound pv-shoot--garden--aws-ha-ff17323b-d62e-4d5e-a742-9de823621490 80Gi RWO gardener.cloud-fast 13d - ``` - Delete all PVCs that are attached to `etcd-main` cluster. - - `kubectl delete pvc -l instance=etcd-main` - -5. Check the etcd's member leases. There should be leases starting with `etcd-main` as many as `etcd-main` replicas. - One of those leases will have holder identity as `:Leader` and rest of etcd member leases have holder identities as `:Member`. - Please ignore the snapshot leases, i.e., those leases which have the suffix `snap`. - - etcd-main member leases: - ``` - NAME HOLDER AGE - etcd-main-0 4c37667312a3912b:Member 1m - etcd-main-1 75a9b74cfd3077cc:Member 1m - etcd-main-2 c62ee6af755e890d:Leader 1m - ``` - - Delete all `etcd-main` member leases. - -6. Edit the `etcd-main` cluster's configmap (ex: `etcd-main-config-4785b0`) as follows: - - Find the `initial-cluster` field in the configmap. It should look similar to the following: - ``` - # Initial cluster - initial-cluster: etcd-main-0=https://etcd-main-0.etcd-main-peer.default.svc:2380,etcd-main-1=https://etcd-main-1.etcd-main-peer.default.svc:2380,etcd-main-2=https://etcd-main-2.etcd-main-peer.default.svc:2380 - ``` - - Change the `initial-cluster` field to have only one member (`etcd-main-0`) in the string. It should now look like this: - - ``` - # Initial cluster - initial-cluster: etcd-main-0=https://etcd-main-0.etcd-main-peer.default.svc:2380 - ``` - -7. Scale up the `etcd-main` statefulset replicas to `1`: - - `kubectl scale sts etcd-main --replicas=1` - -8. Wait for the single-member etcd cluster to be completely ready. - - `kubectl get pods etcd-main-0` will give the following output when ready: - ``` - NAME READY STATUS RESTARTS AGE - etcd-main-0 2/2 Running 0 1m - ``` - -9. Remove the following annotations from the `Etcd` resource `etcd-main`: - - 1. `kubectl annotate etcd etcd-main druid.gardener.cloud/suspend-etcd-spec-reconcile-` - - 2. `kubectl annotate etcd etcd-main druid.gardener.cloud/disable-resource-protection-` - -10. Finally, add the following annotation to the `Etcd` resource `etcd-main`: - - `kubectl annotate etcd etcd-main gardener.cloud/operation='reconcile'` - -11. Verify that the etcd cluster is formed correctly. - - All the `etcd-main` pods will have outputs similar to following: - ``` - NAME READY STATUS RESTARTS AGE - etcd-main-0 2/2 Running 0 5m - etcd-main-1 2/2 Running 0 1m - etcd-main-2 2/2 Running 0 1m - ``` - Additionally, check if the Etcd CR is ready with `kubectl get etcd etcd-main`: - ``` - NAME READY AGE - etcd-main true 13d - ``` - - Additionally, check the leases for 30 seconds at least. There should be leases starting with `etcd-main` as many as `etcd-main` replicas. One of those leases will have holder identity as `:Leader` and rest of those leases have holder identities as `:Member`. The `AGE` of those leases can also be inspected to identify if those leases were updated in conjunction with the restart of the Etcd cluster: Example: - - ``` - NAME HOLDER AGE - etcd-main-0 4c37667312a3912b:Member 1m - etcd-main-1 75a9b74cfd3077cc:Member 1m - etcd-main-2 c62ee6af755e890d:Leader 1m - ``` - diff --git a/internal/component/configmap/configmap_test.go b/internal/component/configmap/configmap_test.go index 15cc25a97..85dc03ffa 100644 --- a/internal/component/configmap/configmap_test.go +++ b/internal/component/configmap/configmap_test.go @@ -371,7 +371,7 @@ func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actu } func getAdvertiseURLsInterface(etcd *druidv1alpha1.Etcd, commType, scheme string) map[string]interface{} { - advertiseUrlsMap := getAdvertiseUrlsMap(etcd, commType, scheme, druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta)) + advertiseUrlsMap := getAdvertiseURLs(etcd, commType, scheme, druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta)) advertiseUrlsInterface := make(map[string]interface{}, len(advertiseUrlsMap)) for podName, urlList := range advertiseUrlsMap { urlsListInterface := make([]interface{}, len(urlList)) diff --git a/internal/component/configmap/etcdconfig.go b/internal/component/configmap/etcdconfig.go index 5133d66a4..e7527f5e3 100644 --- a/internal/component/configmap/etcdconfig.go +++ b/internal/component/configmap/etcdconfig.go @@ -79,8 +79,8 @@ func createEtcdConfig(etcd *druidv1alpha1.Etcd) *etcdConfig { AutoCompactionRetention: ptr.Deref(etcd.Spec.Common.AutoCompactionRetention, defaultAutoCompactionRetention), ListenPeerUrls: fmt.Sprintf("%s://0.0.0.0:%d", peerScheme, ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer)), ListenClientUrls: fmt.Sprintf("%s://0.0.0.0:%d", clientScheme, ptr.Deref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient)), - AdvertisePeerUrls: getAdvertiseUrlsMap(etcd, commTypePeer, peerScheme, peerSvcName), - AdvertiseClientUrls: getAdvertiseUrlsMap(etcd, commTypeClient, clientScheme, peerSvcName), + AdvertisePeerUrls: getAdvertiseURLs(etcd, commTypePeer, peerScheme, peerSvcName), + AdvertiseClientUrls: getAdvertiseURLs(etcd, commTypeClient, clientScheme, peerSvcName), } if peerSecurityConfig != nil { cfg.PeerSecurity = *peerSecurityConfig @@ -93,19 +93,17 @@ func createEtcdConfig(etcd *druidv1alpha1.Etcd) *etcdConfig { } func getSnapshotCount(etcd *druidv1alpha1.Etcd) int64 { - snapshotCount := defaultSnapshotCount if etcd.Spec.Etcd.SnapshotCount != nil { - snapshotCount = *etcd.Spec.Etcd.SnapshotCount + return *etcd.Spec.Etcd.SnapshotCount } - return snapshotCount + return defaultSnapshotCount } func getDBQuotaBytes(etcd *druidv1alpha1.Etcd) int64 { - dbQuotaBytes := defaultDBQuotaBytes if etcd.Spec.Etcd.Quota != nil { - dbQuotaBytes = etcd.Spec.Etcd.Quota.Value() + return etcd.Spec.Etcd.Quota.Value() } - return dbQuotaBytes + return defaultDBQuotaBytes } func getSchemeAndSecurityConfig(tlsConfig *druidv1alpha1.TLSConfig, caPath, serverTLSPath string) (string, *securityConfig) { @@ -133,7 +131,7 @@ func prepareInitialCluster(etcd *druidv1alpha1.Etcd, peerScheme string) string { return strings.Trim(builder.String(), ",") } -func getAdvertiseUrlsMap(etcd *druidv1alpha1.Etcd, commType, scheme, peerSvcName string) advertiseURLs { +func getAdvertiseURLs(etcd *druidv1alpha1.Etcd, commType, scheme, peerSvcName string) advertiseURLs { var port int32 switch commType { case commTypePeer: From 34979303a554cbf6a13bf20e7fb3a4a9cd832f6b Mon Sep 17 00:00:00 2001 From: Anvesh Reddy Pinnapureddy Date: Thu, 7 Nov 2024 22:30:53 +0530 Subject: [PATCH 5/8] update recovery from permanent quorum loss docs --- docs/usage/recovering-etcd-clusters.md | 73 ++++++++++++++++++++++---- 1 file changed, 62 insertions(+), 11 deletions(-) diff --git a/docs/usage/recovering-etcd-clusters.md b/docs/usage/recovering-etcd-clusters.md index 27ffd285d..e74a99002 100644 --- a/docs/usage/recovering-etcd-clusters.md +++ b/docs/usage/recovering-etcd-clusters.md @@ -6,9 +6,9 @@ For a multi-node `Etcd` cluster quorum loss can either be `Transient` or `Perman ## Transient quorum loss -If quorum is lost through transient network failures (e.g. n/w partitions), spike in resource usage which results in OOM, `etcd` automatically and safely resumes (once the network recovers or the resource consumption has come down) and restores quorum. In other cases like transient power loss, etcd persists the Raft log to disk and replays the log to the point of failure and resumes cluster operation. +If quorum is lost through transient network failures (e.g. n/w partitions), spike in resource usage which results in OOM, `etcd` automatically and safely resumes (once the network recovers or the resource consumption has come down) and restores quorum. In other cases like transient power loss, etcd persists the Raft log to disk and replays the log to the point of failure and resumes cluster operation. -## Permanent quorum loss +## Permanent quorum loss In case the quorum is lost due to hardware failures or disk corruption etc, automatic recovery is no longer possible and it is categorized as a permanent quorum loss. @@ -43,6 +43,7 @@ Identify the etcd-cluster which has a permanent quorum loss. Most of the resourc To ensure that only one actor (in this case an operator) makes changes to the `Etcd` resource and also to the `Etcd` cluster resources, following must be done: Add the annotation to the `Etcd` resource: + ```bash kubectl annotate etcd -n druid.gardener.cloud/suspend-etcd-spec-reconcile= ``` @@ -74,6 +75,7 @@ kubectl delete pvc -l instance= -n For a `n` member `Etcd` cluster there should be `n` member `Lease` objects. The lease names should start with the `Etcd` name. Example leases for a 3 node `Etcd` cluster: + ```b NAME HOLDER AGE -0 4c37667312a3912b:Member 1m @@ -82,6 +84,7 @@ Example leases for a 3 node `Etcd` cluster: ``` Delete all the member leases. + ```bash kubectl delete lease # Alternatively you can use label selector. From v0.23.0 onwards leases will have common set of labels @@ -90,18 +93,66 @@ kubectl delete lease -l app.kubernetes.io.component=etcd-member-lease, app.kuber #### 05-Modify ConfigMap -Prerequisite to scale up etcd-cluster from 0->1 is to change `initial-cluster` in the ConfigMap. Assuming that prior to scale-down to 0, there were 3 members, the `initial-cluster` field would look like the following (assuming that the name of the etcd resource is `etcd-main`): +Prerequisite to scale up etcd-cluster from 0->1 is to change the fields `initial-cluster`, `initial-advertise-peer-urls`, and `advertise-client-urls` in the ConfigMap. + +Assuming that prior to scale-down to 0, there were 3 members: + +The `initial-cluster` field would look like the following (assuming that the name of the etcd resource is `etcd-main`): + ```yaml # Initial cluster initial-cluster: etcd-main-0=https://etcd-main-0.etcd-main-peer.default.svc:2380,etcd-main-1=https://etcd-main-1.etcd-main-peer.default.svc:2380,etcd-main-2=https://etcd-main-2.etcd-main-peer.default.svc:2380 ``` -Change the `initial-cluster` field to have only one member (in this case `etc-main-0`). After the change it should look like: -```bash +Change the `initial-cluster` field to have only one member (in this case `etcd-main-0`). After the change it should look like: + +```yaml # Initial cluster initial-cluster: etcd-main-0=https://etcd-main-0.etcd-main-peer.default.svc:2380 ``` +The `initial-advertise-peer-urls` field would look like the following: + +```yaml +# Initial advertise peer urls +initial-advertise-peer-urls: + etcd-main-0: + - http://etcd-main-0.etcd-main-peer.default.svc:2380 + etcd-main-1: + - http://etcd-main-1.etcd-main-peer.default.svc:2380 + etcd-main-2: + - http://etcd-main-2.etcd-main-peer.default.svc:2380 +``` + +Change the `initial-advertise-peer-urls` field to have only one member (in this case `etcd-main-0`). After the change it should look like: + +```yaml +# Initial advertise peer urls +initial-advertise-peer-urls: + etcd-main-0: + - http://etcd-main-0.etcd-main-peer.default.svc:2380 +``` + +The `advertise-client-urls` field would look like the following: + +```yaml +advertise-client-urls: + etcd-main-0: + - http://etcd-main-0.etcd-main-peer.default.svc:2379 + etcd-main-1: + - http://etcd-main-1.etcd-main-peer.default.svc:2379 + etcd-main-2: + - http://etcd-main-2.etcd-main-peer.default.svc:2379 +``` + +Change the `advertise-client-urls` field to have only one member (in this case `etcd-main-0`). After the change it should look like: + +```yaml +advertise-client-urls: + etcd-main-0: + - http://etcd-main-0.etcd-main-peer.default.svc:2379 +``` + #### 06-Scale up Etcd cluster to size 1 ```bash @@ -111,6 +162,7 @@ kubectl scale sts -n --replicas=1 #### 07-Wait for Single-Member etcd cluster to be completely ready To check if the `single-member` etcd cluster is ready check the status of the pod. + ```bash kubectl get pods -n NAME READY STATUS RESTARTS AGE @@ -122,6 +174,7 @@ If both containers report readiness (as seen above), then the etcd-cluster is co #### 08-Enable Etcd reconciliation and resource protection All manual changes are now done. We must now re-enable etcd-cluster resource protection and also enable reconciliation by etcd-druid by doing the following: + ```bash kubectl annotate etcd -n druid.gardener.cloud/suspend-etcd-spec-reconcile- kubectl annotate etcd -n druid.gardener.cloud/disable-etcd-component-protection- @@ -136,8 +189,9 @@ kubectl scale sts -n namespace --replicas=3 ``` If etcd-druid has been set up with `--enable-etcd-spec-auto-reconcile` switched-off then to ensure reconciliation one must annotate `Etcd` resource with the following command: + ```bash -# Annotate etcd-test CR to reconcile +# Annotate etcd CR to reconcile kubectl annotate etcd -n gardener.cloud/operation="reconcile" ``` @@ -154,6 +208,7 @@ NAME READY STATUS RESTARTS AGE ``` Additionally, check if the `Etcd` CR is ready: + ```bash kubectl get etcd -n NAME READY AGE @@ -161,14 +216,10 @@ NAME READY AGE ``` Check member leases, whose `holderIdentity` should reflect the member role. Check if all members are voting members (their role should either be `Member` or `Leader`). Monitor the leases for some time and check if the leases are getting updated. You can monitor the `AGE` field. + ```bash NAME HOLDER AGE -0 4c37667312a3912b:Member 1m -1 75a9b74cfd3077cc:Member 1m -2 c62ee6af755e890d:Leader 1m ``` - - - - - From 254b23d88e2a23a3c8ff880774f7aa7ed84d7bff Mon Sep 17 00:00:00 2001 From: Anvesh Reddy Pinnapureddy Date: Mon, 11 Nov 2024 10:56:33 +0530 Subject: [PATCH 6/8] Address review comments --- api/v1alpha1/helper.go | 2 +- .../03-scaling-up-an-etcd-cluster.md | 2 +- docs/usage/recovering-etcd-clusters.md | 2 +- .../component/configmap/configmap_test.go | 10 ++++---- internal/component/configmap/etcdconfig.go | 24 +++++++++---------- 5 files changed, 19 insertions(+), 21 deletions(-) diff --git a/api/v1alpha1/helper.go b/api/v1alpha1/helper.go index 068c34f70..1cf3252f3 100644 --- a/api/v1alpha1/helper.go +++ b/api/v1alpha1/helper.go @@ -31,7 +31,7 @@ func GetServiceAccountName(etcdObjMeta metav1.ObjectMeta) string { // GetConfigMapName returns the name of the configmap for the Etcd. func GetConfigMapName(etcdObjMeta metav1.ObjectMeta) string { - return fmt.Sprintf("%s-config-%s", etcdObjMeta.Name, (etcdObjMeta.UID[:6])) + return fmt.Sprintf("%s-config-%s", etcdObjMeta.Name, etcdObjMeta.UID[:6]) } // GetCompactionJobName returns the compaction job name for the Etcd. diff --git a/docs/proposals/03-scaling-up-an-etcd-cluster.md b/docs/proposals/03-scaling-up-an-etcd-cluster.md index 3de353360..8f966fb59 100644 --- a/docs/proposals/03-scaling-up-an-etcd-cluster.md +++ b/docs/proposals/03-scaling-up-an-etcd-cluster.md @@ -24,7 +24,7 @@ Now, it is detected whether peer URL was TLS enabled or not for single node etcd - If peer URL was not TLS enabled then etcd-druid has to intervene and make sure peer URL should be TLS enabled first for the single node before marking the cluster for scale-up. ## Action taken by etcd-druid to enable the peerURL TLS -1. Etcd-druid will update the `{etcd.Name}-config` config-map with new config like initial-cluster,initial-advertise-peer-urls etc. Backup-restore will detect this change and update the member lease annotation to `member.etcd.gardener.cloud/tls-enabled: "true"`. +1. Etcd-druid will update the `{etcd.Name}-config-{etcdUID}` config-map with new config like initial-cluster,initial-advertise-peer-urls etc. Backup-restore will detect this change and update the member lease annotation to `member.etcd.gardener.cloud/tls-enabled: "true"`. 2. In case the peer URL TLS has been changed to `enabled`: Etcd-druid will add tasks to the deployment flow: - Check if peer TLS has been enabled for existing StatefulSet pods, by checking the member leases for the annotation `member.etcd.gardener.cloud/tls-enabled`. - If peer TLS enablement is pending for any of the members, then check and patch the StatefulSet with the peer TLS volume mounts, if not already patched. This will cause a rolling update of the existing StatefulSet pods, which allows etcd-backup-restore to update the member peer URL in the etcd cluster. diff --git a/docs/usage/recovering-etcd-clusters.md b/docs/usage/recovering-etcd-clusters.md index e74a99002..9d8f4a995 100644 --- a/docs/usage/recovering-etcd-clusters.md +++ b/docs/usage/recovering-etcd-clusters.md @@ -6,7 +6,7 @@ For a multi-node `Etcd` cluster quorum loss can either be `Transient` or `Perman ## Transient quorum loss -If quorum is lost through transient network failures (e.g. n/w partitions), spike in resource usage which results in OOM, `etcd` automatically and safely resumes (once the network recovers or the resource consumption has come down) and restores quorum. In other cases like transient power loss, etcd persists the Raft log to disk and replays the log to the point of failure and resumes cluster operation. +If quorum is lost through transient network failures (e.g. n/w partitions) or there is a spike in resource usage which results in OOM, `etcd` automatically and safely resumes (once the network recovers or the resource consumption has come down) and restores quorum. In other cases like transient power loss, etcd persists the Raft log to disk and replays the log to the point of failure and resumes cluster operation. ## Permanent quorum loss diff --git a/internal/component/configmap/configmap_test.go b/internal/component/configmap/configmap_test.go index 85dc03ffa..0a49c749e 100644 --- a/internal/component/configmap/configmap_test.go +++ b/internal/component/configmap/configmap_test.go @@ -353,7 +353,7 @@ func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actu if etcd.Spec.Etcd.ClientUrlTLS != nil { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", ptr.Deref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), - "advertise-client-urls": Equal(getAdvertiseURLsInterface(etcd, commTypeClient, "https")), + "advertise-client-urls": Equal(convertAdvertiseURLsValuesToInterface(etcd, advertiseURLTypeClient, "https")), "client-transport-security": MatchKeys(IgnoreExtras, Keys{ "cert-file": Equal("/var/etcd/ssl/server/tls.crt"), "key-file": Equal("/var/etcd/ssl/server/tls.key"), @@ -370,8 +370,8 @@ func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actu } } -func getAdvertiseURLsInterface(etcd *druidv1alpha1.Etcd, commType, scheme string) map[string]interface{} { - advertiseUrlsMap := getAdvertiseURLs(etcd, commType, scheme, druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta)) +func convertAdvertiseURLsValuesToInterface(etcd *druidv1alpha1.Etcd, advertiseURLType, scheme string) map[string]interface{} { + advertiseUrlsMap := getAdvertiseURLs(etcd, advertiseURLType, scheme, druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta)) advertiseUrlsInterface := make(map[string]interface{}, len(advertiseUrlsMap)) for podName, urlList := range advertiseUrlsMap { urlsListInterface := make([]interface{}, len(urlList)) @@ -394,12 +394,12 @@ func matchPeerTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actual "auto-tls": Equal(false), }), "listen-peer-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), - "initial-advertise-peer-urls": Equal(getAdvertiseURLsInterface(etcd, commTypePeer, "https")), + "initial-advertise-peer-urls": Equal(convertAdvertiseURLsValuesToInterface(etcd, advertiseURLTypePeer, "https")), })) } else { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ "listen-peer-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), - "initial-advertise-peer-urls": Equal(getAdvertiseURLsInterface(etcd, commTypePeer, "http")), + "initial-advertise-peer-urls": Equal(convertAdvertiseURLsValuesToInterface(etcd, advertiseURLTypePeer, "http")), })) g.Expect(actualETCDConfig).ToNot(HaveKey("peer-transport-security")) } diff --git a/internal/component/configmap/etcdconfig.go b/internal/component/configmap/etcdconfig.go index e7527f5e3..69fe3b247 100644 --- a/internal/component/configmap/etcdconfig.go +++ b/internal/component/configmap/etcdconfig.go @@ -22,17 +22,15 @@ const ( defaultInitialClusterToken = "etcd-cluster" defaultInitialClusterState = "new" // For more information refer to https://etcd.io/docs/v3.4/op-guide/maintenance/#raft-log-retention - defaultSnapshotCount = int64(75000) - commTypePeer = "peer" // communication type - commTypeClient = "client" + defaultSnapshotCount = int64(75000) + advertiseURLTypePeer = "peer" + advertiseURLTypeClient = "client" ) var ( defaultDataDir = fmt.Sprintf("%s/new.etcd", common.VolumeMountPathEtcdData) ) -type advertiseURLs map[string][]string - type etcdConfig struct { Name string `yaml:"name"` DataDir string `yaml:"data-dir"` @@ -47,8 +45,8 @@ type etcdConfig struct { AutoCompactionRetention string `yaml:"auto-compaction-retention"` ListenPeerUrls string `yaml:"listen-peer-urls"` ListenClientUrls string `yaml:"listen-client-urls"` - AdvertisePeerUrls advertiseURLs `yaml:"initial-advertise-peer-urls"` - AdvertiseClientUrls advertiseURLs `yaml:"advertise-client-urls"` + AdvertisePeerUrls map[string][]string `yaml:"initial-advertise-peer-urls"` + AdvertiseClientUrls map[string][]string `yaml:"advertise-client-urls"` ClientSecurity securityConfig `yaml:"client-transport-security,omitempty"` PeerSecurity securityConfig `yaml:"peer-transport-security,omitempty"` } @@ -79,8 +77,8 @@ func createEtcdConfig(etcd *druidv1alpha1.Etcd) *etcdConfig { AutoCompactionRetention: ptr.Deref(etcd.Spec.Common.AutoCompactionRetention, defaultAutoCompactionRetention), ListenPeerUrls: fmt.Sprintf("%s://0.0.0.0:%d", peerScheme, ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer)), ListenClientUrls: fmt.Sprintf("%s://0.0.0.0:%d", clientScheme, ptr.Deref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient)), - AdvertisePeerUrls: getAdvertiseURLs(etcd, commTypePeer, peerScheme, peerSvcName), - AdvertiseClientUrls: getAdvertiseURLs(etcd, commTypeClient, clientScheme, peerSvcName), + AdvertisePeerUrls: getAdvertiseURLs(etcd, advertiseURLTypePeer, peerScheme, peerSvcName), + AdvertiseClientUrls: getAdvertiseURLs(etcd, advertiseURLTypeClient, clientScheme, peerSvcName), } if peerSecurityConfig != nil { cfg.PeerSecurity = *peerSecurityConfig @@ -131,12 +129,12 @@ func prepareInitialCluster(etcd *druidv1alpha1.Etcd, peerScheme string) string { return strings.Trim(builder.String(), ",") } -func getAdvertiseURLs(etcd *druidv1alpha1.Etcd, commType, scheme, peerSvcName string) advertiseURLs { +func getAdvertiseURLs(etcd *druidv1alpha1.Etcd, advertiseURLType, scheme, peerSvcName string) map[string][]string { var port int32 - switch commType { - case commTypePeer: + switch advertiseURLType { + case advertiseURLTypePeer: port = ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer) - case commTypeClient: + case advertiseURLTypeClient: port = ptr.Deref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient) default: return nil From 812165469edb18671c2a61201e345ed12d52e5df Mon Sep 17 00:00:00 2001 From: Anvesh Reddy Pinnapureddy Date: Thu, 21 Nov 2024 15:07:03 +0530 Subject: [PATCH 7/8] Address review comments --- api/v1alpha1/helper.go | 2 +- api/v1alpha1/helper_test.go | 2 +- .../03-scaling-up-an-etcd-cluster.md | 2 +- .../component/configmap/configmap_test.go | 22 +++++++++++++++++-- internal/component/configmap/etcdconfig.go | 2 +- test/e2e/etcd_backup_test.go | 4 ++-- 6 files changed, 26 insertions(+), 8 deletions(-) diff --git a/api/v1alpha1/helper.go b/api/v1alpha1/helper.go index 1cf3252f3..0bdaed5bd 100644 --- a/api/v1alpha1/helper.go +++ b/api/v1alpha1/helper.go @@ -31,7 +31,7 @@ func GetServiceAccountName(etcdObjMeta metav1.ObjectMeta) string { // GetConfigMapName returns the name of the configmap for the Etcd. func GetConfigMapName(etcdObjMeta metav1.ObjectMeta) string { - return fmt.Sprintf("%s-config-%s", etcdObjMeta.Name, etcdObjMeta.UID[:6]) + return fmt.Sprintf("%s-config", etcdObjMeta.Name) } // GetCompactionJobName returns the compaction job name for the Etcd. diff --git a/api/v1alpha1/helper_test.go b/api/v1alpha1/helper_test.go index a35fbabe7..015b6bb74 100644 --- a/api/v1alpha1/helper_test.go +++ b/api/v1alpha1/helper_test.go @@ -54,7 +54,7 @@ func TestGetConfigMapName(t *testing.T) { uid := uuid.NewUUID() etcdObjMeta := createEtcdObjectMetadata(uid, nil, nil, false) configMapName := GetConfigMapName(etcdObjMeta) - g.Expect(configMapName).To(Equal(etcdObjMeta.Name + "-config-" + string(uid[:6]))) + g.Expect(configMapName).To(Equal(etcdObjMeta.Name + "-config")) } func TestGetCompactionJobName(t *testing.T) { diff --git a/docs/proposals/03-scaling-up-an-etcd-cluster.md b/docs/proposals/03-scaling-up-an-etcd-cluster.md index 8f966fb59..3de353360 100644 --- a/docs/proposals/03-scaling-up-an-etcd-cluster.md +++ b/docs/proposals/03-scaling-up-an-etcd-cluster.md @@ -24,7 +24,7 @@ Now, it is detected whether peer URL was TLS enabled or not for single node etcd - If peer URL was not TLS enabled then etcd-druid has to intervene and make sure peer URL should be TLS enabled first for the single node before marking the cluster for scale-up. ## Action taken by etcd-druid to enable the peerURL TLS -1. Etcd-druid will update the `{etcd.Name}-config-{etcdUID}` config-map with new config like initial-cluster,initial-advertise-peer-urls etc. Backup-restore will detect this change and update the member lease annotation to `member.etcd.gardener.cloud/tls-enabled: "true"`. +1. Etcd-druid will update the `{etcd.Name}-config` config-map with new config like initial-cluster,initial-advertise-peer-urls etc. Backup-restore will detect this change and update the member lease annotation to `member.etcd.gardener.cloud/tls-enabled: "true"`. 2. In case the peer URL TLS has been changed to `enabled`: Etcd-druid will add tasks to the deployment flow: - Check if peer TLS has been enabled for existing StatefulSet pods, by checking the member leases for the annotation `member.etcd.gardener.cloud/tls-enabled`. - If peer TLS enablement is pending for any of the members, then check and patch the StatefulSet with the peer TLS volume mounts, if not already patched. This will cause a rolling update of the existing StatefulSet pods, which allows etcd-backup-restore to update the member peer URL in the etcd cluster. diff --git a/internal/component/configmap/configmap_test.go b/internal/component/configmap/configmap_test.go index 0a49c749e..1ca36931c 100644 --- a/internal/component/configmap/configmap_test.go +++ b/internal/component/configmap/configmap_test.go @@ -334,7 +334,7 @@ func matchConfigMap(g *WithT, etcd *druidv1alpha1.Etcd, actualConfigMap corev1.C err := yaml.Unmarshal([]byte(actualETCDConfigYAML), &actualETCDConfig) g.Expect(err).ToNot(HaveOccurred()) g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ - "name": Equal(fmt.Sprintf("etcd-%s", etcd.UID[:6])), + "name": Equal("etcd-config"), "data-dir": Equal(fmt.Sprintf("%s/new.etcd", common.VolumeMountPathEtcdData)), "metrics": Equal(string(druidv1alpha1.Basic)), "snapshot-count": Equal(ptr.Deref(etcd.Spec.Etcd.SnapshotCount, defaultSnapshotCount)), @@ -370,8 +370,26 @@ func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actu } } +func assertAdvertiseURLs(etcd *druidv1alpha1.Etcd, advertiseURLType, scheme string) map[string][]string { + var port int32 + switch advertiseURLType { + case advertiseURLTypePeer: + port = ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer) + case advertiseURLTypeClient: + port = ptr.Deref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient) + default: + return nil + } + advUrlsMap := make(map[string][]string) + for i := 0; i < int(etcd.Spec.Replicas); i++ { + podName := druidv1alpha1.GetOrdinalPodName(etcd.ObjectMeta, i) + advUrlsMap[podName] = []string{fmt.Sprintf("%s://%s.%s.%s.svc:%d", scheme, podName, druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta), etcd.Namespace, port)} + } + return advUrlsMap +} + func convertAdvertiseURLsValuesToInterface(etcd *druidv1alpha1.Etcd, advertiseURLType, scheme string) map[string]interface{} { - advertiseUrlsMap := getAdvertiseURLs(etcd, advertiseURLType, scheme, druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta)) + advertiseUrlsMap := assertAdvertiseURLs(etcd, advertiseURLType, scheme) advertiseUrlsInterface := make(map[string]interface{}, len(advertiseUrlsMap)) for podName, urlList := range advertiseUrlsMap { urlsListInterface := make([]interface{}, len(urlList)) diff --git a/internal/component/configmap/etcdconfig.go b/internal/component/configmap/etcdconfig.go index 69fe3b247..d15caa3a7 100644 --- a/internal/component/configmap/etcdconfig.go +++ b/internal/component/configmap/etcdconfig.go @@ -64,7 +64,7 @@ func createEtcdConfig(etcd *druidv1alpha1.Etcd) *etcdConfig { peerScheme, peerSecurityConfig := getSchemeAndSecurityConfig(etcd.Spec.Etcd.PeerUrlTLS, common.VolumeMountPathEtcdPeerCA, common.VolumeMountPathEtcdPeerServerTLS) peerSvcName := druidv1alpha1.GetPeerServiceName(etcd.ObjectMeta) cfg := &etcdConfig{ - Name: fmt.Sprintf("etcd-%s", etcd.UID[:6]), + Name: "etcd-config", DataDir: defaultDataDir, Metrics: ptr.Deref(etcd.Spec.Etcd.Metrics, druidv1alpha1.Basic), SnapshotCount: getSnapshotCount(etcd), diff --git a/test/e2e/etcd_backup_test.go b/test/e2e/etcd_backup_test.go index 9dd840c2e..6e4956da5 100644 --- a/test/e2e/etcd_backup_test.go +++ b/test/e2e/etcd_backup_test.go @@ -247,7 +247,7 @@ func checkEtcdReady(ctx context.Context, cl client.Client, logger logr.Logger, e logger.Info("Checking configmap") cm := &corev1.ConfigMap{} - ExpectWithOffset(2, cl.Get(ctx, client.ObjectKey{Name: etcd.Name + "-config-" + string(etcd.UID[:6]), Namespace: etcd.Namespace}, cm)).To(Succeed()) + ExpectWithOffset(2, cl.Get(ctx, client.ObjectKey{Name: etcd.Name + "-config", Namespace: etcd.Namespace}, cm)).To(Succeed()) logger.Info("Checking client service") svc := &corev1.Service{} @@ -280,7 +280,7 @@ func deleteAndCheckEtcd(ctx context.Context, cl client.Client, logger logr.Logge ExpectWithOffset(1, cl.Get( ctx, - client.ObjectKey{Name: etcd.Name + "-config-" + string(etcd.UID[:6]), Namespace: etcd.Namespace}, + client.ObjectKey{Name: etcd.Name + "-config", Namespace: etcd.Namespace}, &corev1.ConfigMap{}, ), ).Should(matchers.BeNotFoundError()) From 5ec17355061571316b6f652e219f2761dfe5ba3f Mon Sep 17 00:00:00 2001 From: Anvesh Reddy Pinnapureddy Date: Fri, 22 Nov 2024 12:40:56 +0530 Subject: [PATCH 8/8] Address review comments by @shreyas-s-rao --- docs/api-reference/etcd-druid-api.md | 1 + internal/component/configmap/configmap_test.go | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/api-reference/etcd-druid-api.md b/docs/api-reference/etcd-druid-api.md index 14824674a..8bded3a8d 100644 --- a/docs/api-reference/etcd-druid-api.md +++ b/docs/api-reference/etcd-druid-api.md @@ -245,6 +245,7 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | | `quota` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.29/#quantity-resource-api)_ | Quota defines the etcd DB quota. | | | +| `snapshotCount` _integer_ | SnapshotCount defines the number of applied Raft entries to hold in-memory before compaction.
More info: https://etcd.io/docs/v3.4/op-guide/maintenance/#raft-log-retention | | | | `defragmentationSchedule` _string_ | DefragmentationSchedule defines the cron standard schedule for defragmentation of etcd. | | | | `serverPort` _integer_ | | | | | `clientPort` _integer_ | | | | diff --git a/internal/component/configmap/configmap_test.go b/internal/component/configmap/configmap_test.go index 1ca36931c..be1408203 100644 --- a/internal/component/configmap/configmap_test.go +++ b/internal/component/configmap/configmap_test.go @@ -353,7 +353,7 @@ func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actu if etcd.Spec.Etcd.ClientUrlTLS != nil { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ "listen-client-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", ptr.Deref(etcd.Spec.Etcd.ClientPort, common.DefaultPortEtcdClient))), - "advertise-client-urls": Equal(convertAdvertiseURLsValuesToInterface(etcd, advertiseURLTypeClient, "https")), + "advertise-client-urls": Equal(expectedAdvertiseURLsAsInterface(etcd, advertiseURLTypeClient, "https")), "client-transport-security": MatchKeys(IgnoreExtras, Keys{ "cert-file": Equal("/var/etcd/ssl/server/tls.crt"), "key-file": Equal("/var/etcd/ssl/server/tls.key"), @@ -370,7 +370,7 @@ func matchClientTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actu } } -func assertAdvertiseURLs(etcd *druidv1alpha1.Etcd, advertiseURLType, scheme string) map[string][]string { +func expectedAdvertiseURLs(etcd *druidv1alpha1.Etcd, advertiseURLType, scheme string) map[string][]string { var port int32 switch advertiseURLType { case advertiseURLTypePeer: @@ -388,8 +388,8 @@ func assertAdvertiseURLs(etcd *druidv1alpha1.Etcd, advertiseURLType, scheme stri return advUrlsMap } -func convertAdvertiseURLsValuesToInterface(etcd *druidv1alpha1.Etcd, advertiseURLType, scheme string) map[string]interface{} { - advertiseUrlsMap := assertAdvertiseURLs(etcd, advertiseURLType, scheme) +func expectedAdvertiseURLsAsInterface(etcd *druidv1alpha1.Etcd, advertiseURLType, scheme string) map[string]interface{} { + advertiseUrlsMap := expectedAdvertiseURLs(etcd, advertiseURLType, scheme) advertiseUrlsInterface := make(map[string]interface{}, len(advertiseUrlsMap)) for podName, urlList := range advertiseUrlsMap { urlsListInterface := make([]interface{}, len(urlList)) @@ -412,12 +412,12 @@ func matchPeerTLSRelatedConfiguration(g *WithT, etcd *druidv1alpha1.Etcd, actual "auto-tls": Equal(false), }), "listen-peer-urls": Equal(fmt.Sprintf("https://0.0.0.0:%d", ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), - "initial-advertise-peer-urls": Equal(convertAdvertiseURLsValuesToInterface(etcd, advertiseURLTypePeer, "https")), + "initial-advertise-peer-urls": Equal(expectedAdvertiseURLsAsInterface(etcd, advertiseURLTypePeer, "https")), })) } else { g.Expect(actualETCDConfig).To(MatchKeys(IgnoreExtras|IgnoreMissing, Keys{ "listen-peer-urls": Equal(fmt.Sprintf("http://0.0.0.0:%d", ptr.Deref(etcd.Spec.Etcd.ServerPort, common.DefaultPortEtcdPeer))), - "initial-advertise-peer-urls": Equal(convertAdvertiseURLsValuesToInterface(etcd, advertiseURLTypePeer, "http")), + "initial-advertise-peer-urls": Equal(expectedAdvertiseURLsAsInterface(etcd, advertiseURLTypePeer, "http")), })) g.Expect(actualETCDConfig).ToNot(HaveKey("peer-transport-security")) }