Refactor worker restart to prevent issues with periodic tests

Author: aequitas

docker/cron-docker/periodic/15min/restart_nassl_worker

@@ -1,4 +1,10 @@
 #!/bin/sh
 set -e
-# find nassl worker and restart the container(s)
-docker ps --filter label=com.docker.compose.service=worker-nassl --quiet |  xargs --no-run-if-empty docker restart
+# stop and start worker one at a time to ensure (batch) tasks are still being picked up
+# workers are sent a TERM signal with which a 10 minute grace period before QUIT is sent
+for worker in $(docker ps --filter label=com.docker.compose.service=worker-nassl --quiet); do
+  docker stop "$worker"
+  docker start "$worker"
+  # wait for container to be healthy
+  timeout 300 sh -c "while docker inspect \"$worker\"| jq --exit-status '.[0].State.Health.Status != \"healthy\"' >/dev/null;do sleep 1;done" || true
+done

docker/cron-docker/periodic/daily/restart_slow_worker

@@ -21,4 +21,4 @@ $COMPOSE_CMD up --no-deps --no-recreate --wait --scale="$SERVICE=$(($REPLICAS*2)
 docker rm --force "$OLD_CONTAINERS"
 
 # restore replica number to original
-$COMPOSE_CMD scale $SERVICE=$REPLICAS
+$COMPOSE_CMD up --no-deps --no-recreate --wait --scale="$SERVICE=$REPLICAS" "$SERVICE"

docker/cron-docker/periodic/hourly/restart_worker

@@ -1,4 +1,10 @@
 #!/bin/sh
 set -e
-# find worker and restart the container(s)
-docker ps --filter label=com.docker.compose.service=worker --quiet | xargs --no-run-if-empty docker restart
+# stop and start worker one at a time to ensure (batch) tasks are still being picked up
+# workers are sent a TERM signal with which a 10 minute grace period before QUIT is sent
+for worker in $(docker ps --filter label=com.docker.compose.service=worker --quiet); do
+  docker stop "$worker"
+  docker start "$worker"
+  # wait for container to be healthy
+  timeout 300 sh -c "while docker inspect \"$worker\"| jq --exit-status '.[0].State.Health.Status != \"healthy\"' >/dev/null;do sleep 1;done" || true
+done

docker/docker-compose.yml

@@ -249,6 +249,8 @@ services:
     # time after which a SIGKILL is sent to celery after a SIGTERM (warm shutdown), default 10s
     # insufficient short grace period causes issues on batch when tasks are killed during the hourly worker restart
     stop_grace_period: 10m
+    # SIGTERM is default, but make it explicit
+    stop_signal: SIGTERM
 
     depends_on:
       db-migrate:
@@ -736,6 +738,8 @@ services:
     environment:
       - AUTO_UPDATE_TO
       - WORKER_SLOW_REPLICAS
+      - WORKER_REPLICAS
+      - RELEASE
 
     restart: unless-stopped
     logging: