Retry apt-get update on failure (GoogleCloudDataproc#169)

Author: karth295

datalab/datalab.sh

@@ -18,13 +18,23 @@
 
 set -e -x
 
+function update_apt_get() {
+  for ((i = 0; i < 10; i++)); do
+    if apt-get update; then
+      return 0
+    fi
+    sleep 5
+  done
+  return 1
+}
+
 ROLE=$(/usr/share/google/get_metadata_value attributes/dataproc-role)
 PROJECT=$(/usr/share/google/get_metadata_value ../project/project-id)
 DOCKER_IMAGE=$(/usr/share/google/get_metadata_value attributes/docker-image || true)
 SPARK_PACKAGES=$(/usr/share/google/get_metadata_value attributes/spark-packages || true)
 
 if [[ "${ROLE}" == 'Master' ]]; then
-  apt-get update
+  update_apt_get
   apt-get install -y -q docker.io
   if [[ -z "${DOCKER_IMAGE}" ]]; then
     DOCKER_IMAGE="gcr.io/cloud-datalab/datalab:local"

flink/flink.sh

@@ -24,6 +24,16 @@
 
 set -euxo pipefail
 
+function update_apt_get() {
+  for ((i = 0; i < 10; i++)); do
+    if apt-get update; then
+      return 0
+    fi
+    sleep 5
+  done
+  return 1
+}
+
 # Install directories for Flink and Hadoop
 readonly FLINK_INSTALL_DIR='/usr/lib/flink'
 readonly HADOOP_CONF_DIR='/etc/hadoop/conf'
@@ -121,6 +131,7 @@ EOF
 function main() {
 local role="$(/usr/share/google/get_metadata_value attributes/dataproc-role)"
 if [[ "${role}" == 'Master' ]] ; then
+  update_apt_get
   apt-get install -y flink
   configure_flink
 fi

ganglia/ganglia.sh

@@ -15,7 +15,18 @@
 
 set -x -e
 
-apt-get update && apt-get install -y ganglia-monitor
+function update_apt_get() {
+  for ((i = 0; i < 10; i++)); do
+    if apt-get update; then
+      return 0
+    fi
+    sleep 5
+  done
+  return 1
+}
+
+update_apt_get
+apt-get install -y ganglia-monitor
 
 MASTER=$(/usr/share/google/get_metadata_value attributes/dataproc-master)
 

hue/hue.sh

@@ -14,6 +14,16 @@
 
 set -x -e
 
+function update_apt_get() {
+  for ((i = 0; i < 10; i++)); do
+    if apt-get update; then
+      return 0
+    fi
+    sleep 5
+  done
+  return 1
+}
+
 # Determine the role of this node
 ROLE=$(/usr/share/google/get_metadata_value attributes/dataproc-role)
 
@@ -23,7 +33,7 @@ if [[ "${ROLE}" != 'Master' ]]; then
 fi
 
 # Install hue
-apt-get update
+update_apt_get
 apt-get install -t jessie-backports hue -y
 
 # Stop hue

jupyter/internal/bootstrap-jupyter-ext.sh

@@ -1,5 +1,17 @@
 #!/bin/bash
 set -e
+
+function update_apt_get() {
+  for ((i = 0; i < 10; i++)); do
+    if apt-get update; then
+      return 0
+    fi
+    sleep 5
+  done
+  return 1
+}
+
+update_apt_get
 sudo apt-get install -y git
 
 # Installs Jupyter notebook extensions and RISE

kafka/kafka.sh

@@ -14,6 +14,17 @@
 #    limitations under the License.
 set -x -e
 
+function update_apt_get() {
+  for ((i = 0; i < 10; i++)); do
+    if apt-get update; then
+      return 0
+    fi
+    sleep 5
+  done
+  return 1
+}
+
+update_apt_get
 
 # Only run the installation on workers; verify zookeeper on master(s)
 if [[ $(/usr/share/google/get_metadata_value attributes/dataproc-role) == 'Master' ]]; then

oozie/oozie.sh

@@ -25,6 +25,16 @@
 
 set -x -e
 
+function update_apt_get() {
+  for ((i = 0; i < 10; i++)); do
+    if apt-get update; then
+      return 0
+    fi
+    sleep 5
+  done
+  return 1
+}
+
 # Determine the role of this node
 ROLE=$(/usr/share/google/get_metadata_value attributes/dataproc-role)
 
@@ -34,7 +44,7 @@ if [[ "${ROLE}" != 'Master' ]]; then
 fi
 
 # Upgrade the repository and install Oozie
-apt-get update
+update_apt_get
 apt-get install oozie oozie-client -y
 
 # The ext library is needed to enable the Oozie web console

tez/tez.sh

@@ -14,6 +14,16 @@
 #  limitations under the License.
 set -x -e
 
+function update_apt_get() {
+  for ((i = 0; i < 10; i++)); do
+    if apt-get update; then
+      return 0
+    fi
+    sleep 5
+  done
+  return 1
+}
+
 #  Define important variables
 tez_version="0.7.0"
 protobuf_version="2.5.0"
@@ -34,6 +44,7 @@ if [[ "${role}" == 'Master' ]]; then
   # Install Tez and YARN Application Timeline Server.
   # Install node/npm to run HTTP server for Tez UI.
   curl -sL https://deb.nodesource.com/setup_8.x | bash -
+  update_apt_get
   apt-get install tez hadoop-yarn-timelineserver nodejs -y
 
   # Stage Tez

user-environment/user-environment.sh

@@ -17,12 +17,22 @@ set -o errexit
 set -o nounset
 set -o xtrace
 
+function update_apt_get() {
+  for ((i = 0; i < 10; i++)); do
+    if apt-get update; then
+      return 0
+    fi
+    sleep 5
+  done
+  return 1
+}
+
 ## Only install customize master node by default.
 ## Delete to customize all nodes.
-[[ "${HOSTNAME}" =~ -m$ ]] || exit
+[[ "${HOSTNAME}" =~ -m$ ]] || exit 0
 
 ## Make global changes here
-apt-get update || true
+update_apt_get
 apt-get install -y vim
 update-alternatives --set editor /usr/bin/vim.basic
 #apt-get install -y tmux sl

zeppelin/zeppelin.sh

@@ -19,14 +19,24 @@
 
 set -x -e
 
+function update_apt_get() {
+  for ((i = 0; i < 10; i++)); do
+    if apt-get update; then
+      return 0
+    fi
+    sleep 5
+  done
+  return 1
+}
+
 # Only run on the master node
 ROLE="$(/usr/share/google/get_metadata_value attributes/dataproc-role)"
 INTERPRETER_FILE='/etc/zeppelin/conf/interpreter.json'
 ZEPPELIN_PORT="$(/usr/share/google/get_metadata_value attributes/zeppelin-port || true)"
 
 if [[ "${ROLE}" == 'Master' ]]; then
   # Install zeppelin. Don't mind if it fails to start the first time.
-  apt-get update || true
+  update_apt_get
   apt-get install -y -t jessie-backports zeppelin || dpkg -l zeppelin
 
   for i in {1..6}; do