fix(setup): Ensure built-in users exist before proceeding

Fixes #786

setup/entrypoint.sh

@@ -3,7 +3,7 @@
 set -eu
 set -o pipefail
 
-source "$(dirname "${BASH_SOURCE[0]}")/helpers.sh"
+source "${BASH_SOURCE[0]%/*}"/helpers.sh
 
 
 # --------------------------------------------------------
@@ -33,7 +33,7 @@ roles_files=(
 
 echo "-------- $(date) --------"
 
-state_file="$(dirname "${BASH_SOURCE[0]}")/state/.done"
+state_file="${BASH_SOURCE[0]%/*}"/state/.done
 if [[ -e "$state_file" ]]; then
  log "State file exists at '${state_file}', skipping setup"
  exit 0
@@ -65,11 +65,22 @@ fi
 
 sublog 'Elasticsearch is running'
 
+log 'Waiting for initialization of built-in users'
+
+wait_for_builtin_users || exit_code=$?
+
+if ((exit_code)); then
+ suberr 'Timed out waiting for condition'
+ exit $exit_code
+fi
+
+sublog 'Built-in users were initialized'
+
 for role in "${!roles_files[@]}"; do
  log "Role '$role'"
 
  declare body_file
- body_file="$(dirname "${BASH_SOURCE[0]}")/roles/${roles_files[$role]:-}"
+ body_file="${BASH_SOURCE[0]%/*}/roles/${roles_files[$role]:-}"
  if [[ ! -f "${body_file:-}" ]]; then
  sublog "No role body found at '${body_file}', skipping"
  continue
@@ -94,7 +105,7 @@ for user in "${!users_passwords[@]}"; do
  set_user_password "$user" "${users_passwords[$user]}"
  else
  if [[ -z "${users_roles[$user]:-}" ]]; then
- err ' No role defined, skipping creation'
+ suberr ' No role defined, skipping creation'
  continue
  fi
 
@@ -103,5 +114,5 @@ for user in "${!users_passwords[@]}"; do
  fi
 done
 
-mkdir -p "$(dirname "${state_file}")"
+mkdir -p "${state_file%/*}"
 touch "$state_file"

setup/helpers.sh

@@ -57,6 +57,53 @@ function wait_for_elasticsearch {
  return $result
 }
 
+# Poll the Elasticsearch users API until it returns users.
+function wait_for_builtin_users {
+ local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
+
+ local -a args=( '-s' '-D-' '-m15' "http://${elasticsearch_host}:9200/_security/user?pretty" )
+
+ if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
+ args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
+ fi
+
+ local -i result=1
+
+ local line
+ local -i exit_code
+ local -i num_users
+
+ # retry for max 30s (30*1s)
+ for _ in $(seq 1 30); do
+ num_users=0
+
+ # read exits with a non-zero code if the last read input doesn't end
+ # with a newline character. The printf without newline that follows the
+ # curl command ensures that the final input not only contains curl's
+ # exit code, but causes read to fail so we can capture the return value.
+ # Ref. https://unix.stackexchange.com/a/176703/152409
+ while IFS= read -r line || ! exit_code="$line"; do
+ if [[ "$line" =~ _reserved.+true ]]; then
+ (( num_users++ ))
+ fi
+ done < <(curl "${args[@]}"; printf '%s' "$?")
+
+ if ((exit_code)); then
+ result=$exit_code
+ fi
+
+ # we expect more than just the 'elastic' user in the result
+ if (( num_users > 1 )); then
+ result=0
+ break
+ fi
+
+ sleep 1
+ done
+
+ return $result
+}
+
 # Verify that the given Elasticsearch user exists.
 function check_user_exists {
  local username=$1