New Dockerfile and init.sh for the api

api/Dockerfile

@@ -1,36 +1,59 @@
 FROM python:3.11.6-slim-bookworm
 
-ARG ENVIRONMENT
+ENV PYTHONUNBUFFERED=1
+
+ENV RUN_MIGRATION=yes
+ENV RUN_COMPILE_MESSAGES=no
+ENV RUN_COLLECT_STATIC=no
+ENV RUN_CREATE_SUPER_USER=yes
+
+ENV IS_CONTAINERIZED=True
+
+ENV DEBIAN_FRONTEND=noninteractive
 
-ENV ENVIRONMENT ${ENVIRONMENT:-production}
-ENV DJANGO_SETTINGS_MODULE seismic_site.settings.${ENVIRONMENT}
-ENV DOCKERIZE_VERSION v0.6.1
 
 RUN apt-get update && \
+    apt-get upgrade -y && \
     apt-get install -y --no-install-recommends \
-        build-essential \
-        postgresql-client \
-        libpq-dev \
-        gettext \
-        wget
+    nginx gcc xz-utils gettext build-essential postgresql-client libpq-dev
+
+
+ARG S6_OVERLAY_VERSION=3.1.2.1
+ENV S6_CMD_WAIT_FOR_SERVICES_MAXTIME 0
+
+ADD https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-noarch.tar.xz /tmp
+ADD https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-x86_64.tar.xz /tmp
+RUN tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz && \
+    tar -C / -Jxpf /tmp/s6-overlay-x86_64.tar.xz
+
+
+ENTRYPOINT ["/init"]
+
+COPY docker/nginx/nginx.conf /etc/nginx/sites-available/default
+COPY docker/s6-rc.d /etc/s6-overlay/s6-rc.d
+
+
+# Python virtualenv paths
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+
+# install the backend libraries:
+COPY ./api/setup.cfg ./api/requirements.txt ./api/requirements-dev.txt /var/www/seismic/api/
+WORKDIR /var/www/seismic/api/
+RUN python3 -m pip install --upgrade pip setuptools && \
+    python3 -m pip install -r ./requirements.txt
+
 
-# install dockerize and update pip
-RUN wget https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz && \
-    tar -C /usr/local/bin -xzvf dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz && \
-    rm dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz && \
-    chown root:root /usr/local/bin/dockerize && \
-    pip install "pip==23.*" -U
+# copy the backend source code:
+COPY ./api/ /var/www/seismic/api/
 
-# install and cache app dependencies
-WORKDIR /opt/
 
-COPY ./requirements*.txt ./
-RUN if [ "${ENVIRONMENT}" = "production" ]; then pip install -r requirements.txt; else pip install -r requirements-dev.txt; fi
+# Make sure scripts in .local are usable:
+ENV PATH=/root/.local/bin:$PATH
 
-WORKDIR /code/
 
-COPY ./docker-entrypoint /
-COPY ./ /code/
+WORKDIR /var/www/seismic/
 
-ENTRYPOINT ["/docker-entrypoint"]
-EXPOSE 8030
+EXPOSE 80

api/Dockerfile.old

@@ -0,0 +1,36 @@
+FROM python:3.11.6-slim-bookworm
+
+ARG ENVIRONMENT
+
+ENV ENVIRONMENT ${ENVIRONMENT:-production}
+ENV DJANGO_SETTINGS_MODULE seismic_site.settings.${ENVIRONMENT}
+ENV DOCKERIZE_VERSION v0.6.1
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends \
+        build-essential \
+        postgresql-client \
+        libpq-dev \
+        gettext \
+        wget
+
+# install dockerize and update pip
+RUN wget https://github.com/jwilder/dockerize/releases/download/${DOCKERIZE_VERSION}/dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz && \
+    tar -C /usr/local/bin -xzvf dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz && \
+    rm dockerize-linux-amd64-${DOCKERIZE_VERSION}.tar.gz && \
+    chown root:root /usr/local/bin/dockerize && \
+    pip install "pip==23.*" -U
+
+# install and cache app dependencies
+WORKDIR /opt/
+
+COPY ./requirements*.txt ./
+RUN if [ "${ENVIRONMENT}" = "production" ]; then pip install -r requirements.txt; else pip install -r requirements-dev.txt; fi
+
+WORKDIR /code/
+
+COPY ./docker-entrypoint /
+COPY ./ /code/
+
+ENTRYPOINT ["/docker-entrypoint"]
+EXPOSE 8030

api/requirements-dev.txt

@@ -4,8 +4,6 @@
 #
 #    pip-compile --output-file=requirements-dev.txt requirements-dev.in
 #
-appnope==0.1.3
-    # via ipython
 asgiref==3.7.2
     # via
     #   -r requirements.txt

api/requirements.in

@@ -9,6 +9,12 @@ django-jquery==3.1.0
 django-import-export==3.3.1
 django-jazzmin==2.6.0
 
+# job scheduler
+django-q2~=1.5.5
+blessed~=1.20.0  # optional requirement for django-q2
+psutil~=5.9.5  # optional requirement for django-q2
+croniter~=1.4.1  # optional requirement for django-q2
+
 # REST framework
 djangorestframework==3.14.0
 drf-spectacular==0.26.5

api/requirements.txt

@@ -10,12 +10,16 @@ attrs==23.1.0
     # via
     #   jsonschema
     #   referencing
+blessed==1.20.0
+    # via -r requirements.in
 boto3==1.28.61
     # via -r requirements.in
 botocore==1.31.61
     # via
     #   boto3
     #   s3transfer
+croniter==1.4.1
+    # via -r requirements.in
 defusedxml==0.7.1
     # via odfpy
 diff-match-patch==20230430
@@ -30,6 +34,8 @@ django==4.2.6
     #   django-import-export
     #   django-jazzmin
     #   django-jquery
+    #   django-picklefield
+    #   django-q2
     #   django-storages
     #   djangorestframework
     #   drf-spectacular
@@ -43,6 +49,10 @@ django-jazzmin==2.6.0
     # via -r requirements.in
 django-jquery==3.1.0
     # via -r requirements.in
+django-picklefield==3.1
+    # via django-q2
+django-q2==1.5.5
+    # via -r requirements.in
 django-storages==1.14.2
     # via -r requirements.in
 djangorestframework==3.14.0
@@ -79,10 +89,14 @@ packaging==23.2
     # via gunicorn
 pillow==10.0.1
     # via -r requirements.in
+psutil==5.9.5
+    # via -r requirements.in
 psycopg2==2.9.9
     # via -r requirements.in
 python-dateutil==2.8.2
-    # via botocore
+    # via
+    #   botocore
+    #   croniter
 pytz==2023.3.post1
     # via djangorestframework
 pyyaml==6.0.1
@@ -100,7 +114,9 @@ rpds-py==0.10.4
 s3transfer==0.7.0
     # via boto3
 six==1.16.0
-    # via python-dateutil
+    # via
+    #   blessed
+    #   python-dateutil
 sqlparse==0.4.4
     # via django
 tablib[html,ods,xls,xlsx,yaml]==3.5.0
@@ -113,6 +129,8 @@ uritemplate==4.1.1
     # via drf-spectacular
 urllib3==1.26.17
     # via botocore
+wcwidth==0.2.8
+    # via blessed
 whitenoise==6.5.0
     # via -r requirements.in
 xlrd==2.0.1

api/seismic_site/settings/base.py

@@ -41,6 +41,7 @@
     SUPER_ADMIN_EMAIL=(str, ""),
     SUPER_ADMIN_FIRST_NAME=(str, ""),
     SUPER_ADMIN_LAST_NAME=(str, ""),
+    BACKGROUND_WORKERS=(int, 1),
 )
 
 DATA_UPLOAD_MAX_NUMBER_FIELDS = env.int("DATA_UPLOAD_MAX_NUMBER_FIELDS")
@@ -72,6 +73,7 @@
     "rest_framework",
     "storages",
     "corsheaders",
+    "django_q",
     # project apps
     "buildings",
     "static_custom",
@@ -463,3 +465,26 @@
         "success": "btn-outline-success",
     },
 }
+
+
+# Django Q2
+# https://django-q2.readthedocs.io/en/stable/brokers.html
+
+Q_CLUSTER = {
+    "name": "seismic",
+    "workers": env.int("BACKGROUND_WORKERS"),
+    "recycle": 100,
+    "timeout": 900,  # A task must finish in less than 15 minutes
+    "retry": 1200,  # Retry an unfinished tasks after 20 minutes
+    "ack_failures": True,
+    "max_attempts": 2,
+    "compress": True,
+    "save_limit": 200,
+    "queue_limit": 4,
+    "cpu_affinity": 1,
+    "label": "Django Q2",
+    "orm": "default",
+    "poll": 2,
+    "guard_cycle": 3,
+    "catch_up": False,
+}

client/Dockerfile

@@ -1,5 +1,5 @@
 # base image
-FROM node:18-alpine
+FROM node:18.18-bookworm-slim
 
 ARG ENVIRONMENT
 

docker-compose.yml

@@ -14,14 +14,14 @@ services:
   api:
     container_name: seismic-api
     build:
-      context: ./api
-    command: dockerize -wait tcp://db:5444 -timeout 90s
+      context: .
+      dockerfile: ./api/Dockerfile
     env_file:
       - .env
     volumes:
       - ./api:/code
     ports:
-      - "8030:8030"
+      - "8030:80"
     depends_on:
       - db
 

docker/nginx/nginx.conf

@@ -0,0 +1,16 @@
+server {
+    listen [::]:80 default_server;
+    listen 80 default_server;
+    server_name _;
+
+    client_max_body_size 200M;
+
+    location  / {
+        # proxy_set_header X-Forwarded-Proto https;
+        proxy_set_header X-Url-Scheme $scheme;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host $http_host;
+        proxy_redirect off;
+        proxy_pass http://unix:/run/gunicorn.sock;
+    }
+}

docker/s6-rc.d/api/dependencies

@@ -0,0 +1 @@
+init

docker/s6-rc.d/api/run

@@ -0,0 +1,12 @@
+#!/command/with-contenv sh
+cd /var/www/seismic/api/
+
+# https://docs.gunicorn.org/en/latest/design.html#how-many-workers
+WORKERS=$(((2 * $(nproc)) + 1))
+
+python3 -m gunicorn seismic_site.wsgi \
+    --bind "unix:///run/gunicorn.sock" \
+    --log-level info \
+    --worker-class gevent \
+    --workers $WORKERS \
+    --timeout 60

docker/s6-rc.d/api/type

@@ -0,0 +1 @@
+longrun

docker/s6-rc.d/init/init.sh

@@ -0,0 +1,45 @@
+#!/command/with-contenv bash
+
+cd "${BACKEND_ROOT:-/var/www/seismic/api}" || exit 1
+
+echo "Checking..."
+python3 manage.py check
+
+if [ "${RUN_MIGRATION^^}" = "TRUE" ]; then
+    echo "Migrating database"
+    python3 manage.py migrate --run-syncdb
+    python3 manage.py createcachetable
+fi
+
+if [ "${RUN_COMPILE_MESSAGES^^}" = "TRUE" ]; then
+    echo "Compiling translation messages"
+    python3 manage.py compilemessages
+fi
+
+if [ "${RUN_COLLECT_STATIC^^}" = "TRUE" ]; then
+    echo "Collect static"
+    mkdir static
+    python3 manage.py collectstatic --noinput
+fi
+
+if [ "${RUN_CREATE_SUPER_USER^^}" = "TRUE" ]; then
+    echo "Check superuser presence"
+    SUPERUSERS=$(python3 manage.py shell -c "from django.contrib.auth import get_user_model; User = get_user_model(); print(User.objects.filter(username=\"${DJANGO_ADMIN_EMAIL}\").count())")
+
+    if [ "${SUPERUSERS}" = "0" ]; then
+        echo "Create first superuser"
+        python3 manage.py createsuperuser --noinput --username "${DJANGO_ADMIN_EMAIL}"
+
+        echo "Set superuser password"
+        python3 manage.py shell -c "from django.contrib.auth import get_user_model; User = get_user_model(); u = User.objects.get(username=\"${DJANGO_ADMIN_EMAIL}\"); u.set_password(\"${DJANGO_ADMIN_PASSWORD}\"); u.save()"
+    else
+        echo "A superuser already exists"
+    fi
+fi
+
+if [ "${RUN_LOAD_DUMMY_DATA^^}" = "TRUE" ] ; then
+  echo "Load dummy data into the database"
+  ./manage.py loaddata proximal_utilities
+  ./manage.py loaddata work_performed
+fi
+

docker/s6-rc.d/init/type

@@ -0,0 +1 @@
+oneshot

docker/s6-rc.d/init/up

@@ -0,0 +1 @@
+/etc/s6-overlay/s6-rc.d/init/init.sh

docker/s6-rc.d/nginx/dependencies

@@ -0,0 +1 @@
+api

docker/s6-rc.d/nginx/run

@@ -0,0 +1,2 @@
+#!/bin/sh -e
+nginx -g "daemon off;"

docker/s6-rc.d/nginx/type

@@ -0,0 +1 @@
+longrun

docker/s6-rc.d/qcluster/dependencies

@@ -0,0 +1 @@
+init

docker/s6-rc.d/qcluster/run

@@ -0,0 +1,4 @@
+#!/command/with-contenv sh
+cd /var/www/seismic/api/
+
+python3 manage.py qcluster

docker/s6-rc.d/qcluster/type

@@ -0,0 +1 @@
+longrun