Merge pull request #6141 from mailcow/staging

2024-11

.github/workflows/check_prs_if_on_staging.yml

@@ -10,7 +10,7 @@ jobs:
     if: github.event.pull_request.base.ref != 'staging' #check if the target branch is not staging
     steps:
       - name: Send message
-        uses: thollander/actions-comment-pull-request@v2.5.0
+        uses: thollander/actions-comment-pull-request@v3.0.1
         with:
           GITHUB_TOKEN: ${{ secrets.CHECKIFPRISSTAGING_ACTION_PAT }}
           message: |

.github/workflows/update_postscreen_access_list.yml

@@ -22,7 +22,7 @@ jobs:
           bash helper-scripts/update_postscreen_whitelist.sh
 
     - name: Create Pull Request
-      uses: peter-evans/create-pull-request@v6
+      uses: peter-evans/create-pull-request@v7
       with:
         token: ${{ secrets.mailcow_action_Update_postscreen_access_cidr_pat }}
         commit-message: update postscreen_access.cidr

.gitignore

@@ -45,6 +45,7 @@ data/conf/rspamd/override.d/*
 data/conf/sogo/custom-theme.js
 data/conf/sogo/plist_ldap
 data/conf/sogo/sieve.creds
+data/conf/sogo/cron.creds
 data/conf/sogo/sogo-full.svg
 data/gitea/
 data/gogs/

data/Dockerfiles/dockerapi/main.py

@@ -90,7 +90,7 @@ async def get_container(container_id : str):
         if container._id == container_id:
           container_info = await container.show()
           return Response(content=json.dumps(container_info, indent=4), media_type="application/json")
-     
+
       res = {
         "type": "danger",
         "msg": "no container found"
@@ -130,7 +130,7 @@ async def get_containers():
 async def post_containers(container_id : str, post_action : str, request: Request):
   global dockerapi
 
-  try : 
+  try:
     request_json = await request.json()
   except Exception as err:
     request_json = {}
@@ -191,7 +191,7 @@ async def post_container_update_stats(container_id : str):
 
   stats = json.loads(await dockerapi.redis_client.get(container_id + '_stats'))
   return Response(content=json.dumps(stats, indent=4), media_type="application/json")
-  
+
 
 # PubSub Handler
 async def handle_pubsub_messages(channel: aioredis.client.PubSub):
@@ -244,7 +244,7 @@ async def handle_pubsub_messages(channel: aioredis.client.PubSub):
               dockerapi.logger.error("Unknwon PubSub recieved - %s" % json.dumps(data_json))
           else:
             dockerapi.logger.error("Unknwon PubSub recieved - %s" % json.dumps(data_json))
-              
+
         await asyncio.sleep(0.0)
     except asyncio.TimeoutError:
       pass

data/Dockerfiles/dockerapi/modules/DockerApi.py

@@ -159,7 +159,7 @@ def container_post__exec__mailq__deliver(self, request_json, **kwargs):
             postqueue_r = container.exec_run(["/bin/bash", "-c", "/usr/sbin/postqueue " + i], user='postfix')
             # todo: check each exit code
           res = { 'type': 'success', 'msg': 'Scheduled immediate delivery'}
-          return Response(content=json.dumps(res, indent=4), media_type="application/json")        
+          return Response(content=json.dumps(res, indent=4), media_type="application/json")
   # api call: container_post - post_action: exec - cmd: mailq - task: list
   def container_post__exec__mailq__list(self, request_json, **kwargs):
     if 'container_id' in kwargs:
@@ -318,7 +318,7 @@ def container_post__exec__sieve__print(self, request_json, **kwargs):
 
     if 'username' in request_json and 'script_name' in request_json:
       for container in self.sync_docker_client.containers.list(filters=filters):
-        cmd = ["/bin/bash", "-c", "/usr/bin/doveadm sieve get -u '" + request_json['username'].replace("'", "'\\''") + "' '" + request_json['script_name'].replace("'", "'\\''") + "'"]  
+        cmd = ["/bin/bash", "-c", "/usr/bin/doveadm sieve get -u '" + request_json['username'].replace("'", "'\\''") + "' '" + request_json['script_name'].replace("'", "'\\''") + "'"]
         sieve_return = container.exec_run(cmd)
         return self.exec_run_handler('utf8_text_only', sieve_return)
   # api call: container_post - post_action: exec - cmd: maildir - task: cleanup
@@ -342,6 +342,30 @@ def container_post__exec__maildir__cleanup(self, request_json, **kwargs):
           cmd = ["/bin/bash", "-c", cmd_vmail]
         maildir_cleanup = container.exec_run(cmd, user='vmail')
         return self.exec_run_handler('generic', maildir_cleanup)
+  # api call: container_post - post_action: exec - cmd: maildir - task: move
+  def container_post__exec__maildir__move(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    if 'old_maildir' in request_json and 'new_maildir' in request_json:
+      for container in self.sync_docker_client.containers.list(filters=filters):
+        vmail_name = request_json['old_maildir'].replace("'", "'\\''")
+        new_vmail_name = request_json['new_maildir'].replace("'", "'\\''")
+        cmd_vmail = f"if [[ -d '/var/vmail/{vmail_name}' ]]; then /bin/mv '/var/vmail/{vmail_name}' '/var/vmail/{new_vmail_name}'; fi"
+
+        index_name = request_json['old_maildir'].split("/")
+        new_index_name = request_json['new_maildir'].split("/")
+        if len(index_name) > 1 and len(new_index_name) > 1:
+          index_name = index_name[1].replace("'", "'\\''") + "@" + index_name[0].replace("'", "'\\''")
+          new_index_name = new_index_name[1].replace("'", "'\\''") + "@" + new_index_name[0].replace("'", "'\\''")
+          cmd_vmail_index = f"if [[ -d '/var/vmail_index/{index_name}' ]]; then /bin/mv '/var/vmail_index/{index_name}' '/var/vmail_index/{new_index_name}_index'; fi"
+          cmd = ["/bin/bash", "-c", cmd_vmail + " && " + cmd_vmail_index]
+        else:
+          cmd = ["/bin/bash", "-c", cmd_vmail]
+        maildir_move = container.exec_run(cmd, user='vmail')
+        return self.exec_run_handler('generic', maildir_move)
   # api call: container_post - post_action: exec - cmd: rspamd - task: worker_password
   def container_post__exec__rspamd__worker_password(self, request_json, **kwargs):
     if 'container_id' in kwargs:
@@ -374,6 +398,121 @@ def container_post__exec__rspamd__worker_password(self, request_json, **kwargs):
           self.logger.error('failed changing Rspamd password')
           res = { 'type': 'danger', 'msg': 'command did not complete' }
           return Response(content=json.dumps(res, indent=4), media_type="application/json")
+  # api call: container_post - post_action: exec - cmd: sogo - task: rename
+  def container_post__exec__sogo__rename_user(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    if 'old_username' in request_json and 'new_username' in request_json:
+      for container in self.sync_docker_client.containers.list(filters=filters):
+        old_username = request_json['old_username'].replace("'", "'\\''")
+        new_username = request_json['new_username'].replace("'", "'\\''")
+
+        sogo_return = container.exec_run(["/bin/bash", "-c", f"sogo-tool rename-user '{old_username}' '{new_username}'"], user='sogo')
+        return self.exec_run_handler('generic', sogo_return)
+  # api call: container_post - post_action: exec - cmd: doveadm - task: get_acl
+  def container_post__exec__doveadm__get_acl(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      id = request_json['id'].replace("'", "'\\''")
+
+      shared_folders = container.exec_run(["/bin/bash", "-c", f"doveadm mailbox list -u '{id}'"])
+      shared_folders = shared_folders.output.decode('utf-8')
+      shared_folders = shared_folders.splitlines()
+
+      formatted_acls = []
+      mailbox_seen = []
+      for shared_folder in shared_folders:
+        if "Shared" not in shared_folder:
+          mailbox = shared_folder.replace("'", "'\\''")
+          if mailbox in mailbox_seen:
+            continue
+
+          acls = container.exec_run(["/bin/bash", "-c", f"doveadm acl get -u '{id}' '{mailbox}'"])
+          acls = acls.output.decode('utf-8').strip().splitlines()
+          if len(acls) >= 2:
+            for acl in acls[1:]:
+              user_id, rights = acl.split(maxsplit=1)
+              user_id = user_id.split('=')[1]
+              mailbox_seen.append(mailbox)
+              formatted_acls.append({ 'user': id, 'id': user_id, 'mailbox': mailbox, 'rights': rights.split() })
+        elif "Shared" in shared_folder and "/" in shared_folder:
+          shared_folder = shared_folder.split("/")
+          if len(shared_folder) < 3:
+            continue
+
+          user = shared_folder[1].replace("'", "'\\''")
+          mailbox = '/'.join(shared_folder[2:]).replace("'", "'\\''")
+          if mailbox in mailbox_seen:
+            continue
+
+          acls = container.exec_run(["/bin/bash", "-c", f"doveadm acl get -u '{user}' '{mailbox}'"])
+          acls = acls.output.decode('utf-8').strip().splitlines()
+          if len(acls) >= 2:
+            for acl in acls[1:]:
+              user_id, rights = acl.split(maxsplit=1)
+              user_id = user_id.split('=')[1].replace("'", "'\\''")
+              if user_id == id and mailbox not in mailbox_seen:
+                mailbox_seen.append(mailbox)
+                formatted_acls.append({ 'user': user, 'id': id, 'mailbox': mailbox, 'rights': rights.split() })
+
+      return Response(content=json.dumps(formatted_acls, indent=4), media_type="application/json")
+  # api call: container_post - post_action: exec - cmd: doveadm - task: delete_acl
+  def container_post__exec__doveadm__delete_acl(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      user = request_json['user'].replace("'", "'\\''")
+      mailbox = request_json['mailbox'].replace("'", "'\\''")
+      id = request_json['id'].replace("'", "'\\''")
+
+      if user and mailbox and id:
+        acl_delete_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl delete -u '{user}' '{mailbox}' 'user={id}'"])
+        return self.exec_run_handler('generic', acl_delete_return)
+  # api call: container_post - post_action: exec - cmd: doveadm - task: set_acl
+  def container_post__exec__doveadm__set_acl(self, request_json, **kwargs):
+    if 'container_id' in kwargs:
+      filters = {"id": kwargs['container_id']}
+    elif 'container_name' in kwargs:
+      filters = {"name": kwargs['container_name']}
+
+    for container in self.sync_docker_client.containers.list(filters=filters):
+      user = request_json['user'].replace("'", "'\\''")
+      mailbox = request_json['mailbox'].replace("'", "'\\''")
+      id = request_json['id'].replace("'", "'\\''")
+      rights = ""
+
+      available_rights = [
+        "admin",
+        "create",
+        "delete",
+        "expunge",
+        "insert",
+        "lookup",
+        "post",
+        "read",
+        "write",
+        "write-deleted",
+        "write-seen"
+      ]
+      for right in request_json['rights']:
+        right = right.replace("'", "'\\''").lower()
+        if right in available_rights:
+          rights += right + " "
+
+      if user and mailbox and id and rights:
+        acl_set_return = container.exec_run(["/bin/bash", "-c", f"doveadm acl set -u '{user}' '{mailbox}' 'user={id}' {rights}"])
+        return self.exec_run_handler('generic', acl_set_return)
+
 
   # Collect host stats
   async def get_host_stats(self, wait=5):
@@ -462,7 +601,7 @@ def recv_socket_data(c_socket, timeout):
         except:
           pass
       return ''.join(total_data)
-      
+
     try :
       socket = container.exec_run([shell_cmd], stdin=True, socket=True, user=user).output._sock
       if not cmd.endswith("\n"):

data/Dockerfiles/dovecot/docker-entrypoint.sh

@@ -114,15 +114,15 @@ if [[ "${FLATCURVE_EXPERIMENTAL}" =~ ^([yY][eE][sS]|[yY]) ]]; then
 echo -e "\e[33mActivating Flatcurve as FTS Backend...\e[0m"
 echo -e "\e[33mDepending on your previous setup a full reindex might be needed... \e[0m"
 echo -e "\e[34mVisit https://docs.mailcow.email/manual-guides/Dovecot/u_e-dovecot-fts/#fts-related-dovecot-commands to learn how to reindex\e[0m"
-echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify fts fts_flatcurve listescape replication' > /etc/dovecot/mail_plugins
+echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify fts fts_flatcurve listescape replication lazy_expunge' > /etc/dovecot/mail_plugins
 echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify mail_log fts fts_flatcurve listescape replication' > /etc/dovecot/mail_plugins_imap
 echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl fts fts_flatcurve notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
 elif [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
-echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify listescape replication' > /etc/dovecot/mail_plugins
+echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify listescape replication lazy_expunge' > /etc/dovecot/mail_plugins
 echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify listescape replication mail_log' > /etc/dovecot/mail_plugins_imap
 echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
 else
-echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify fts fts_solr listescape replication' > /etc/dovecot/mail_plugins
+echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify fts fts_solr listescape replication lazy_expunge' > /etc/dovecot/mail_plugins
 echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify mail_log fts fts_solr listescape replication' > /etc/dovecot/mail_plugins_imap
 echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl fts fts_solr notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
 fi
@@ -371,6 +371,8 @@ EOF
 # Create random master Password for SOGo SSO
 RAND_PASS=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 32 | head -n 1)
 echo -n ${RAND_PASS} > /etc/phpfpm/sogo-sso.pass
+# Creating additional creds file for SOGo notify crons (calendars, etc)
+echo -n ${RAND_USER}@mailcow.local:${RAND_PASS} > /etc/sogo/cron.creds
 cat <<EOF > /etc/dovecot/sogo-sso.conf
 # Autogenerated by mailcow
 passdb {
@@ -405,6 +407,17 @@ else
 	chown 401 /mail_crypt/ecprivkey.pem /mail_crypt/ecpubkey.pem
 fi
 
+# Fix OpenSSL 3.X TLS1.0, 1.1 support (https://community.mailcow.email/d/4062-hi-all/20)
+if grep -qE 'ssl_min_protocol\s*=\s*(TLSv1|TLSv1\.1)\s*$' /etc/dovecot/dovecot.conf /etc/dovecot/extra.conf; then
+    sed -i '/\[openssl_init\]/a ssl_conf = ssl_configuration' /etc/ssl/openssl.cnf
+
+    echo "[ssl_configuration]" >> /etc/ssl/openssl.cnf
+    echo "system_default = tls_system_default" >> /etc/ssl/openssl.cnf
+    echo "[tls_system_default]" >> /etc/ssl/openssl.cnf
+    echo "MinProtocol = TLSv1" >> /etc/ssl/openssl.cnf
+    echo "CipherString = DEFAULT@SECLEVEL=0" >> /etc/ssl/openssl.cnf
+fi
+
 # Compile sieve scripts
 sievec /var/vmail/sieve/global_sieve_before.sieve
 sievec /var/vmail/sieve/global_sieve_after.sieve

data/Dockerfiles/phpfpm/Dockerfile

@@ -1,17 +1,17 @@
-FROM php:8.2-fpm-alpine3.18
+FROM php:8.2-fpm-alpine3.20
 
 LABEL maintainer = "The Infrastructure Company GmbH <[email protected]>"
 
 # renovate: datasource=github-tags depName=krakjoe/apcu versioning=semver-coerced extractVersion=^v(?<version>.*)$
-ARG APCU_PECL_VERSION=5.1.23
+ARG APCU_PECL_VERSION=5.1.24
 # renovate: datasource=github-tags depName=Imagick/imagick versioning=semver-coerced extractVersion=(?<version>.*)$
 ARG IMAGICK_PECL_VERSION=3.7.0
 # renovate: datasource=github-tags depName=php/pecl-mail-mailparse versioning=semver-coerced extractVersion=^v(?<version>.*)$
-ARG MAILPARSE_PECL_VERSION=3.1.6
+ARG MAILPARSE_PECL_VERSION=3.1.8
 # renovate: datasource=github-tags depName=php-memcached-dev/php-memcached versioning=semver-coerced extractVersion=^v(?<version>.*)$
 ARG MEMCACHED_PECL_VERSION=3.2.0
 # renovate: datasource=github-tags depName=phpredis/phpredis versioning=semver-coerced extractVersion=(?<version>.*)$
-ARG REDIS_PECL_VERSION=6.0.2
+ARG REDIS_PECL_VERSION=6.1.0
 # renovate: datasource=github-tags depName=composer/composer versioning=semver-coerced extractVersion=(?<version>.*)$
 ARG COMPOSER_VERSION=2.6.6
 

data/Dockerfiles/phpfpm/docker-entrypoint.sh

@@ -10,16 +10,25 @@ done
 
 # Do not attempt to write to slave
 if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
-  REDIS_CMDLINE="redis-cli -h ${REDIS_SLAVEOF_IP} -p ${REDIS_SLAVEOF_PORT}"
+  REDIS_HOST=$REDIS_SLAVEOF_IP
+  REDIS_PORT=$REDIS_SLAVEOF_PORT
 else
-  REDIS_CMDLINE="redis-cli -h redis -p 6379"
+  REDIS_HOST="redis"
+  REDIS_PORT="6379"
 fi
+REDIS_CMDLINE="redis-cli -h ${REDIS_HOST} -p ${REDIS_PORT}"
 
 until [[ $(${REDIS_CMDLINE} PING) == "PONG" ]]; do
   echo "Waiting for Redis..."
   sleep 2
 done
 
+# Set redis session store
+echo -n '
+session.save_handler = redis
+session.save_path = "tcp://'${REDIS_HOST}':'${REDIS_PORT}'"
+' > /usr/local/etc/php/conf.d/session_store.ini
+
 # Check mysql_upgrade (master and slave)
 CONTAINER_ID=
 until [[ ! -z "${CONTAINER_ID}" ]] && [[ "${CONTAINER_ID}" =~ ^[[:alnum:]]*$ ]]; do

data/Dockerfiles/postfix/docker-entrypoint.sh

@@ -12,4 +12,15 @@ if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
   cp /etc/syslog-ng/syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng.conf
 fi
 
+# Fix OpenSSL 3.X TLS1.0, 1.1 support (https://community.mailcow.email/d/4062-hi-all/20)
+if grep -qE '\!SSLv2|\!SSLv3|>=TLSv1(\.[0-1])?$' /opt/postfix/conf/main.cf /opt/postfix/conf/extra.cf; then
+    sed -i '/\[openssl_init\]/a ssl_conf = ssl_configuration' /etc/ssl/openssl.cnf
+
+    echo "[ssl_configuration]" >> /etc/ssl/openssl.cnf
+    echo "system_default = tls_system_default" >> /etc/ssl/openssl.cnf
+    echo "[tls_system_default]" >> /etc/ssl/openssl.cnf
+    echo "MinProtocol = TLSv1" >> /etc/ssl/openssl.cnf
+    echo "CipherString = DEFAULT@SECLEVEL=0" >> /etc/ssl/openssl.cnf
+fi  
+
 exec "$@"

data/Dockerfiles/rspamd/Dockerfile

@@ -1,8 +1,8 @@
 FROM debian:bookworm-slim
-LABEL maintainer = "The Infrastructure Company GmbH <[email protected]>"
+LABEL maintainer="The Infrastructure Company GmbH <[email protected]>"
 
 ARG DEBIAN_FRONTEND=noninteractive
-ARG RSPAMD_VER=rspamd_3.9.1-1~82f43560f
+ARG RSPAMD_VER=rspamd_3.10.2-1~b8a232043
 ARG CODENAME=bookworm
 ENV LC_ALL=C
 

data/Dockerfiles/sogo/Dockerfile

@@ -33,13 +33,14 @@ RUN echo "Building from repository $SOGO_DEBIAN_REPOSITORY" \
   && gosu nobody true \
   && mkdir /usr/share/doc/sogo \
   && touch /usr/share/doc/sogo/empty.sh \
-  && apt-key adv --keyserver keys.openpgp.org --recv-key 74FFC6D72B925A34B5D356BDF8A27B36A6E2EAE9 \
+  && wget http://www.axis.cz/linux/debian/axis-archive-keyring.deb -O /tmp/axis-archive-keyring.deb \
+  && apt install -y /tmp/axis-archive-keyring.deb \
   && echo "deb [trusted=yes] ${SOGO_DEBIAN_REPOSITORY} ${DEBIAN_VERSION} sogo-v5" > /etc/apt/sources.list.d/sogo.list \
   && apt-get update && apt-get install -y --no-install-recommends \
     sogo \
     sogo-activesync \
   && apt-get autoclean \
-  && rm -rf /var/lib/apt/lists/* /etc/apt/sources.list.d/sogo.list \
+  && rm -rf /var/lib/apt/lists/* \
   && touch /etc/default/locale
 
 COPY ./bootstrap-sogo.sh /bootstrap-sogo.sh

data/Dockerfiles/sogo/docker-entrypoint.sh

@@ -10,6 +10,8 @@ if [[ ! -z ${REDIS_SLAVEOF_IP} ]]; then
   cp /etc/syslog-ng/syslog-ng-redis_slave.conf /etc/syslog-ng/syslog-ng.conf
 fi
 
+echo "$TZ" > /etc/timezone
+
 # Run hooks
 for file in /hooks/*; do
   if [ -x "${file}" ]; then

data/conf/postfix/main.cf

@@ -170,6 +170,8 @@ smtputf8_enable = no
 submission_smtpd_tls_mandatory_protocols = >=TLSv1.2
 smtps_smtpd_tls_mandatory_protocols = >=TLSv1.2
 parent_domain_matches_subdomains = debug_peer_list,fast_flush_domains,mynetworks,qmqpd_authorized_clients
+# This Option is added to correctly set the X-Original-To Header when mails are send to lmtp (dovecot)
+lmtp_destination_recipient_limit=1
 
 # DO NOT EDIT ANYTHING BELOW #
 # Overrides #