ETCD ON K8S bad member ID arg

[wangchao732]

trconv.ParseUint: parsing "": invalid syntax), expecting ID in Hex

helm:

kind: StatefulSet
apiVersion: apps/v1
metadata:
name: apisix-etcd
namespace: ingress-apisix
labels:
app.kubernetes.io/instance: apisix
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/name: etcd
helm.sh/chart: etcd-6.2.6
annotations:
meta.helm.sh/release-name: apisix
meta.helm.sh/release-namespace: ingress-apisix
spec:
replicas: 3
selector:
matchLabels:
app.kubernetes.io/instance: apisix
app.kubernetes.io/name: etcd
template:
metadata:
creationTimestamp: null
labels:
app.kubernetes.io/instance: apisix
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/name: etcd
helm.sh/chart: etcd-6.2.6
spec:
containers:
- name: etcd
image: 'docker.io/bitnami/etcd:3.4.16-debian-10-r14'
ports:
- name: client
containerPort: 2379
protocol: TCP
- name: peer
containerPort: 2380
protocol: TCP
env:
- name: BITNAMI_DEBUG
value: 'false'
- name: MY_POD_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: MY_POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: ETCDCTL_API
value: '3'
- name: ETCD_ON_K8S
value: 'yes'
- name: ETCD_START_FROM_SNAPSHOT
value: 'no'
- name: ETCD_DISASTER_RECOVERY
value: 'no'
- name: ETCD_NAME
value: $(MY_POD_NAME)
- name: ETCD_DATA_DIR
value: /bitnami/etcd/data
- name: ETCD_LOG_LEVEL
value: info
- name: ALLOW_NONE_AUTHENTICATION
value: 'yes'
- name: ETCD_ADVERTISE_CLIENT_URLS
value: >-
http://$(MY_POD_NAME).apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379
- name: ETCD_LISTEN_CLIENT_URLS
value: 'http://0.0.0.0:2379'
- name: ETCD_INITIAL_ADVERTISE_PEER_URLS
value: >-
http://$(MY_POD_NAME).apisix-etcd-headless.ingress-apisix.svc.cluster.local:2380
- name: ETCD_LISTEN_PEER_URLS
value: 'http://0.0.0.0:2380'
- name: ETCD_INITIAL_CLUSTER_TOKEN
value: etcd-cluster-k8s
- name: ETCD_INITIAL_CLUSTER_STATE
value: existing
- name: ETCD_INITIAL_CLUSTER
value: >-
apisix-etcd-0=http://apisix-etcd-0.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2380,apisix-etcd-1=http://apisix-etcd-1.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2380,apisix-etcd-2=http://apisix-etcd-2.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2380
- name: ETCD_CLUSTER_DOMAIN
value: apisix-etcd-headless.ingress-apisix.svc.cluster.local
resources: {}
volumeMounts:
- name: data
mountPath: /bitnami/etcd
livenessProbe:
exec:
command:
- /opt/bitnami/scripts/etcd/healthcheck.sh
initialDelaySeconds: 60
timeoutSeconds: 5
periodSeconds: 30
successThreshold: 1
failureThreshold: 5
readinessProbe:
exec:
command:
- /opt/bitnami/scripts/etcd/healthcheck.sh
initialDelaySeconds: 60
timeoutSeconds: 5
periodSeconds: 10
successThreshold: 1
failureThreshold: 5
lifecycle:
preStop:
exec:
command:
- /opt/bitnami/scripts/etcd/prestop.sh
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
imagePullPolicy: IfNotPresent
securityContext:
runAsUser: 1001
runAsNonRoot: true
restartPolicy: Always
terminationGracePeriodSeconds: 30
dnsPolicy: ClusterFirst
serviceAccountName: default
serviceAccount: default
securityContext:
fsGroup: 1001
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 1
podAffinityTerm:
labelSelector:
matchLabels:
app.kubernetes.io/instance: apisix
app.kubernetes.io/name: etcd
namespaces:
- ingress-apisix
topologyKey: kubernetes.io/hostname
schedulerName: default-scheduler
volumeClaimTemplates:
- kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: data
creationTimestamp: null
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 8Gi
volumeMode: Filesystem
status:
phase: Pending
serviceName: apisix-etcd-headless
podManagementPolicy: Parallel
updateStrategy:
type: RollingUpdate
revisionHistoryLimit: 10

ERROR POD

kind: Pod
apiVersion: v1
metadata:
name: apisix-etcd-2
generateName: apisix-etcd-
namespace: ingress-apisix
labels:
app.kubernetes.io/instance: apisix
app.kubernetes.io/managed-by: Helm
app.kubernetes.io/name: etcd
controller-revision-hash: apisix-etcd-765dd8cddb
helm.sh/chart: etcd-6.2.6
statefulset.kubernetes.io/pod-name: apisix-etcd-2
annotations:
kubesphere.io/restartedAt: '2023-11-11T10:40:12.941Z'
spec:
volumes:
- name: data
persistentVolumeClaim:
claimName: data-apisix-etcd-2
- name: host-time
hostPath:
path: /etc/localtime
type: ''
- name: kube-api-access-stm7s
projected:
sources:
- serviceAccountToken:
expirationSeconds: 3607
path: token
- configMap:
name: kube-root-ca.crt
items:
- key: ca.crt
path: ca.crt
- downwardAPI:
items:
- path: namespace
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
defaultMode: 420
containers:
- name: etcd
image: 'docker.io/bitnami/etcd:3.4.16-debian-10-r14'
ports:
- name: client
containerPort: 2379
protocol: TCP
- name: peer
containerPort: 2380
protocol: TCP
env:
- name: BITNAMI_DEBUG
value: 'true'
- name: MY_POD_IP
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: status.podIP
- name: MY_POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: ETCDCTL_API
value: '3'
- name: ETCD_ON_K8S
value: 'yes'
- name: ETCD_START_FROM_SNAPSHOT
value: 'no'
- name: ETCD_DISASTER_RECOVERY
value: 'no'
- name: ETCD_NAME
value: $(MY_POD_NAME)
- name: ETCD_DATA_DIR
value: /bitnami/etcd/data
- name: ETCD_LOG_LEVEL
value: debug
- name: ALLOW_NONE_AUTHENTICATION
value: 'yes'
- name: ETCD_ADVERTISE_CLIENT_URLS
value: >-
http://$(MY_POD_NAME).apisix-etcd-headless.ingress-apisix.svc.cluster.local:2379
- name: ETCD_LISTEN_CLIENT_URLS
value: 'http://0.0.0.0:2379'
- name: ETCD_INITIAL_ADVERTISE_PEER_URLS
value: >-
http://$(MY_POD_NAME).apisix-etcd-headless.ingress-apisix.svc.cluster.local:2380
- name: ETCD_LISTEN_PEER_URLS
value: 'http://0.0.0.0:2380'
- name: ETCD_INITIAL_CLUSTER_TOKEN
value: etcd-cluster-k8s
- name: ETCD_INITIAL_CLUSTER_STATE
value: existing
- name: ETCD_INITIAL_CLUSTER
value: >-
apisix-etcd-0=http://apisix-etcd-0.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2380,apisix-etcd-1=http://apisix-etcd-1.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2380,apisix-etcd-2=http://apisix-etcd-2.apisix-etcd-headless.ingress-apisix.svc.cluster.local:2380
- name: ETCD_CLUSTER_DOMAIN
value: apisix-etcd-headless.ingress-apisix.svc.cluster.local
resources: {}
volumeMounts:
- name: data
mountPath: /bitnami/etcd
- name: host-time
readOnly: true
mountPath: /etc/localtime
- name: kube-api-access-stm7s
readOnly: true
mountPath: /var/run/secrets/kubernetes.io/serviceaccount
livenessProbe:
exec:
command:
- /opt/bitnami/scripts/etcd/healthcheck.sh
initialDelaySeconds: 60
timeoutSeconds: 5
periodSeconds: 30
successThreshold: 1
failureThreshold: 5
readinessProbe:
exec:
command:
- /opt/bitnami/scripts/etcd/healthcheck.sh
initialDelaySeconds: 60
timeoutSeconds: 5
periodSeconds: 10
successThreshold: 1
failureThreshold: 5
lifecycle:
preStop:
exec:
command:
- /opt/bitnami/scripts/etcd/prestop.sh
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
imagePullPolicy: IfNotPresent
securityContext:
runAsUser: 1001
runAsNonRoot: true
restartPolicy: Always
terminationGracePeriodSeconds: 30
dnsPolicy: ClusterFirst
serviceAccountName: default
serviceAccount: default
nodeName: bj13-k8s121
securityContext:
fsGroup: 1001
hostname: apisix-etcd-2
subdomain: apisix-etcd-headless
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 1
podAffinityTerm:
labelSelector:
matchLabels:
app.kubernetes.io/instance: apisix
app.kubernetes.io/name: etcd
namespaces:
- ingress-apisix
topologyKey: kubernetes.io/hostname
schedulerName: default-scheduler
tolerations:
- key: node.kubernetes.io/not-ready
operator: Exists
effect: NoExecute
tolerationSeconds: 300
- key: node.kubernetes.io/unreachable
operator: Exists
effect: NoExecute
tolerationSeconds: 300
priority: 0
enableServiceLinks: true
preemptionPolicy: PreemptLowerPriority

[jmhbnz]

Hi @wangchao732 - Thanks for raising this support issue.

It looks like you are using the bitnami helm chart for etcd as part of your apisix deployment. Please refer to the issue below to resolve the issue, it looks like you will need to update to a newer version of the bitnami etcd helm chart:

• Error: bad member ID arg (strconv.ParseUint: parsing "": invalid syntax), expecting ID in Hex bitnami/charts#10909

If you continue to run into issues with this helm chart please raise an issue directly on the bitnami/charts repository.

[wangchao732]

Hi,@jmhbnz
Thank you very much for your reply, the problem is this: there will always be a node unavailable and unable to connect to the cluster on the etcd on k8s 3 replica, and some operations were done earlier, such as kicking the failed node out of the cluster and rebuilding a new node to join the cluster, but encountered problems. The current version is updated to the (image: 'docker.io/bitnami/etcd:3.5.11-debian-11-r2') image version.

error info
Error: etcdclient: no available endpoints

etcd 10:34:47.14 INFO ==>

etcd 10:34:47.14 INFO ==> Welcome to the Bitnami etcd container

etcd 10:34:47.14 INFO ==> Subscribe to project updates by watching https://github.com/bitnami/containers

etcd 10:34:47.14 INFO ==> Submit issues and feature requests at https://github.com/bitnami/containers/issues

etcd 10:34:47.15 INFO ==>

etcd 10:34:47.15 INFO ==> ** Starting etcd setup **

etcd 10:34:47.16 INFO ==> Validating settings in ETCD_* env vars..

etcd 10:34:47.17 WARN ==> You set the environment variable ALLOW_NONE_AUTHENTICATION=yes. For safety reasons, do not use this flag in a production environment.

etcd 10:34:47.17 INFO ==> Initializing etcd

etcd 10:34:47.17 INFO ==> Generating etcd config file using env variables

etcd 10:34:47.19 INFO ==> There is no data from previous deployments

etcd 10:34:47.19 INFO ==> Adding new member to existing cluster

/opt/bitnami/scripts/libetcd.sh: line 269: MY_STS_NAME: unbound variable

/opt/bitnami/scripts/libetcd.sh: line 436: ETCD_ACTIVE_ENDPOINTS: unbound variable

Error: etcdclient: no available endpoints

etcd 10:34:47.24 WARN ==> Failed to add self to cluster, keeping trying...

Error: etcdclient: no available endpoints

etcd 10:34:57.27 WARN ==> Failed to add self to cluster, keeping trying...

Error: etcdclient: no available endpoints

etcd 10:35:07.29 WARN ==> Failed to add self to cluster, keeping trying...

Error: etcdclient: no available endpoints

etcd 10:35:17.31 WARN ==> Failed to add self to cluster, keeping trying...

Error: etcdclient: no available endpoints

etcd 10:35:27.34 WARN ==> Failed to add self to cluster, keeping trying...

Error: etcdclient: no available endpoints

etcd 10:35:37.36 WARN ==> Failed to add self to cluster, keeping trying...

Error: etcdclient: no available endpoints

etcd 10:35:47.39 WARN ==> Failed to add self to cluster, keeping trying...

Error: etcdclient: no available endpoints

etcd 10:35:57.41 WARN ==> Failed to add self to cluster, keeping trying...

Error: etcdclient: no available endpoints

etcd 10:36:07.43 WARN ==> Failed to add self to cluster, keeping trying...

Error: etcdclient: no available endpoints

etcd 10:36:17.46 WARN ==> Failed to add self to cluster, keeping trying...

Error: etcdclient: no available endpoints