add healthcheck

Author: giancarloromeo

services/docker-compose.yml

@@ -1178,7 +1178,7 @@ services:
       S3_ENDPOINT: ${S3_ENDPOINT}
       S3_REGION: ${S3_REGION}
       S3_SECRET_KEY: ${S3_SECRET_KEY}
-      STORAGE_WORKER_MODE: "False"
+      STORAGE_WORKER_MODE: 0
       STORAGE_LOGLEVEL: ${STORAGE_LOGLEVEL}
       STORAGE_MONITORING_ENABLED: 1
       STORAGE_PROFILING: ${STORAGE_PROFILING}
@@ -1193,7 +1193,7 @@ services:
   storage-worker:
     image: ${DOCKER_REGISTRY:-itisfoundation}/storage:${DOCKER_IMAGE_TAG:-latest}
     init: true
-    hostname: "sto-{{.Node.Hostname}}-{{.Task.Slot}}"
+    hostname: "stow-{{.Node.Hostname}}-{{.Task.Slot}}"
     environment:
       BF_API_KEY: ${BF_API_KEY}
       BF_API_SECRET: ${BF_API_SECRET}
@@ -1221,7 +1221,7 @@ services:
       S3_ENDPOINT: ${S3_ENDPOINT}
       S3_REGION: ${S3_REGION}
       S3_SECRET_KEY: ${S3_SECRET_KEY}
-      STORAGE_WORKER_MODE: "True"
+      STORAGE_WORKER_MODE: 1
       STORAGE_LOGLEVEL: ${STORAGE_LOGLEVEL}
       STORAGE_MONITORING_ENABLED: 1
       STORAGE_PROFILING: ${STORAGE_PROFILING}

services/storage/docker/boot.sh

@@ -56,11 +56,12 @@ if [ "${SC_BOOT_MODE}" = "debug" ]; then
       --log-level \"${SERVER_LOG_LEVEL}\"
   "
 else
-  if [ "${STORAGE_WORKER_MODE}" = "True" ]; then
+  if [ "${STORAGE_WORKER_MODE}" = "1" ]; then
     exec celery \
-      --app=simcore_service_storage.modules.celery.worker_main \
+      --app=simcore_service_storage.modules.celery.worker_main:app \
       worker --pool=threads \
-      --loglevel="${SERVER_LOG_LEVEL}"
+      --loglevel="${SERVER_LOG_LEVEL}" \
+      --hostname="${HOSTNAME}"
   else
     exec uvicorn simcore_service_storage.main:app \
       --host 0.0.0.0 \

services/storage/docker/healthcheck.py

@@ -20,18 +20,50 @@
 
 
 import os
+import subprocess
 import sys
 from urllib.request import urlopen
 
+from simcore_service_storage.main import ApplicationSettings
+
 SUCCESS, UNHEALTHY = 0, 1
 
 # Disabled if boots with debugger
-ok = os.environ.get("SC_BOOT_MODE", "").lower() == "debug"
+ok = os.getenv("SC_BOOT_MODE", "").lower() == "debug"
 
 # Queries host
 # pylint: disable=consider-using-with
+app_settings = ApplicationSettings.create_from_envs()
+
+broker_url = app_settings.STORAGE_CELERY_BROKER.dsn
+
+
+def _is_celery_worker_healthy():
+    try:
+        result = subprocess.run(
+            [
+                "celery",
+                "-b",
+                broker_url,
+                "inspect",
+                "ping",
+                "-d",
+                "celery@" + os.getenv("HOSTNAME", ""),
+            ],
+            capture_output=True,
+            text=True,
+            check=True,
+        )
+        return "pong" in result.stdout
+    except subprocess.CalledProcessError:
+        return False
+
+
+print(_is_celery_worker_healthy())
+
 ok = (
     ok
+    or (bool(os.environ.get("STORAGE_WORKER_MODE", "") and _is_celery_worker_healthy()))
     or urlopen(
         "{host}{baseurl}".format(
             host=sys.argv[1], baseurl=os.environ.get("SIMCORE_NODE_BASEPATH", "")

services/storage/src/simcore_service_storage/core/settings.py

@@ -106,7 +106,12 @@ class ApplicationSettings(BaseApplicationSettings, MixinLoggingSettings):
         description="is a dictionary that maps specific loggers (such as 'uvicorn.access' or 'gunicorn.access') to a list of _logger message patterns that should be filtered out.",
     )
 
-    STORAGE_WORKER_MODE: bool | None = False
+    STORAGE_WORKER_MODE: Annotated[
+        bool | None, Field(description="If True, run as a worker")
+    ] = False
+    STORAGE_CELERY_BROKER: RabbitSettings | None = Field(
+        json_schema_extra={"auto_default_from_env": True}
+    )
 
     @field_validator("LOG_LEVEL", mode="before")
     @classmethod