Merge pull request #5922 from mailcow/staging

2024-06

.github/workflows/rebuild_backup_image.yml

@@ -26,7 +26,7 @@ jobs:
           password: ${{ secrets.BACKUPIMAGEBUILD_ACTION_DOCKERHUB_TOKEN }}
 
       - name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/amd64,linux/arm64

data/Dockerfiles/acme/Dockerfile

@@ -1,8 +1,8 @@
-FROM alpine:3.18
+FROM alpine:3.20
 
 LABEL maintainer "The Infrastructure Company GmbH <[email protected]>"
 
-ARG PIP_BREAK_SYSTEM_PACKAGES=1
+
 RUN apk upgrade --no-cache \
   && apk add --update --no-cache \
   bash \
@@ -15,9 +15,7 @@ RUN apk upgrade --no-cache \
   tini \
   tzdata \
   python3 \
-  py3-pip \
-  && pip3 install --upgrade pip \
-  && pip3 install acme-tiny
+  acme-tiny --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community/
 
 COPY acme.sh /srv/acme.sh
 COPY functions.sh /srv/functions.sh

data/Dockerfiles/acme/acme.sh

@@ -33,6 +33,10 @@ if [[ "${ONLY_MAILCOW_HOSTNAME}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
   ONLY_MAILCOW_HOSTNAME=y
 fi
 
+if [[ "${AUTODISCOVER_SAN}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+  AUTODISCOVER_SAN=y
+fi
+
 # Request individual certificate for every domain
 if [[ "${ENABLE_SSL_SNI}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
   ENABLE_SSL_SNI=y
@@ -211,7 +215,11 @@ while true; do
       ADDITIONAL_SAN_ARR+=($i)
     fi
   done
+
+  if [[ ${AUTODISCOVER_SAN} == "y" ]]; then
+  # Fetch certs for autoconfig and autodiscover subdomains
   ADDITIONAL_WC_ARR+=('autodiscover' 'autoconfig')
+  fi
 
   if [[ ${SKIP_IP_CHECK} != "y" ]]; then
   # Start IP detection

data/Dockerfiles/backup/Dockerfile

@@ -1,3 +1,3 @@
-FROM debian:bullseye-slim
+FROM debian:bookworm-slim
 
 RUN apt update && apt install pigz

data/Dockerfiles/clamd/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.19
+FROM alpine:3.20
 
 LABEL maintainer "The Infrastructure Company GmbH <[email protected]>"
 

data/Dockerfiles/dockerapi/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.19
+FROM alpine:3.20
 
 LABEL maintainer "The Infrastructure Company GmbH <[email protected]>"
 

data/Dockerfiles/dockerapi/modules/DockerApi.py

@@ -358,8 +358,8 @@ def container_post__exec__rspamd__worker_password(self, request_json, **kwargs):
         for line in cmd_response.split("\n"):
           if '$2$' in line:
             hash = line.strip()
-            hash_out = re.search('\$2\$.+$', hash).group(0)
-            rspamd_passphrase_hash = re.sub('[^0-9a-zA-Z\$]+', '', hash_out.rstrip())
+            hash_out = re.search(r'\$2\$.+$', hash).group(0)
+            rspamd_passphrase_hash = re.sub(r'[^0-9a-zA-Z\$]+', '', hash_out.rstrip())
             rspamd_password_filename = "/etc/rspamd/override.d/worker-controller-password.inc"
             cmd = '''/bin/echo 'enable_password = "%s";' > %s && cat %s''' % (rspamd_passphrase_hash, rspamd_password_filename, rspamd_password_filename)
             cmd_response = self.exec_cmd_container(container, cmd, user="_rspamd")

data/Dockerfiles/dovecot/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.19
+FROM alpine:3.20
 LABEL maintainer "The Infrastructure Company GmbH <[email protected]>"
 
 # renovate: datasource=github-releases depName=tianon/gosu versioning=semver-coerced extractVersion=^(?<version>.*)$
@@ -24,6 +24,7 @@ RUN addgroup -g 5000 vmail \
   envsubst \
   ca-certificates \
   curl \
+  coreutils \
   jq \
   lua \
   lua-cjson \
@@ -62,7 +63,7 @@ RUN addgroup -g 5000 vmail \
   perl-package-stash-xs \
   perl-par-packer \
   perl-parse-recdescent \
-  perl-lockfile-simple --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community/ \
+  perl-lockfile-simple \
   libproc \
   perl-readonly \
   perl-regexp-common \
@@ -104,13 +105,12 @@ RUN addgroup -g 5000 vmail \
   dovecot-pigeonhole-plugin \
   dovecot-pop3d \
   dovecot-fts-solr \
+  dovecot-fts-flatcurve \
   && arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) \
   && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-$arch" \
   && chmod +x /usr/local/bin/gosu \
   && gosu nobody true
 
-# RUN cpan LockFile::Simple
-
 COPY trim_logs.sh /usr/local/bin/trim_logs.sh
 COPY clean_q_aged.sh /usr/local/bin/clean_q_aged.sh
 COPY syslog-ng.conf /etc/syslog-ng/syslog-ng.conf
@@ -129,6 +129,7 @@ COPY stop-supervisor.sh /usr/local/sbin/stop-supervisor.sh
 COPY quarantine_notify.py /usr/local/bin/quarantine_notify.py
 COPY quota_notify.py /usr/local/bin/quota_notify.py
 COPY repl_health.sh /usr/local/bin/repl_health.sh
+COPY optimize-fts.sh /usr/local/bin/optimize-fts.sh
 
 ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf

data/Dockerfiles/dovecot/docker-entrypoint.sh

@@ -29,6 +29,7 @@ ${REDIS_CMDLINE} SET DOVECOT_REPL_HEALTH 1 > /dev/null
 # Create missing directories
 [[ ! -d /etc/dovecot/sql/ ]] && mkdir -p /etc/dovecot/sql/
 [[ ! -d /etc/dovecot/lua/ ]] && mkdir -p /etc/dovecot/lua/
+[[ ! -d /etc/dovecot/conf.d/ ]] && mkdir -p /etc/dovecot/conf.d/
 [[ ! -d /var/vmail/_garbage ]] && mkdir -p /var/vmail/_garbage
 [[ ! -d /var/vmail/sieve ]] && mkdir -p /var/vmail/sieve
 [[ ! -d /etc/sogo ]] && mkdir -p /etc/sogo
@@ -109,7 +110,14 @@ EOF
 
 echo -n ${ACL_ANYONE} > /etc/dovecot/acl_anyone
 
-if [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+if [[ "${FLATCURVE_EXPERIMENTAL}" =~ ^([yY][eE][sS]|[yY]) ]]; then
+echo -e "\e[33mActivating Flatcurve as FTS Backend...\e[0m"
+echo -e "\e[33mDepending on your previous setup a full reindex might be needed... \e[0m"
+echo -e "\e[34mVisit https://docs.mailcow.email/manual-guides/Dovecot/u_e-dovecot-fts/#fts-related-dovecot-commands to learn how to reindex\e[0m"
+echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify fts fts_flatcurve listescape replication' > /etc/dovecot/mail_plugins
+echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify mail_log fts fts_flatcurve listescape replication' > /etc/dovecot/mail_plugins_imap
+echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl fts fts_flatcurve notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
+elif [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
 echo -n 'quota acl zlib mail_crypt mail_crypt_acl mail_log notify listescape replication' > /etc/dovecot/mail_plugins
 echo -n 'quota imap_quota imap_acl acl zlib imap_zlib imap_sieve mail_crypt mail_crypt_acl notify listescape replication mail_log' > /etc/dovecot/mail_plugins_imap
 echo -n 'quota sieve acl zlib mail_crypt mail_crypt_acl notify listescape replication' > /etc/dovecot/mail_plugins_lmtp
@@ -239,6 +247,47 @@ function script_deinit()
 end
 EOF
 
+# Temporarily set FTS depending on user choice inside mailcow.conf. Will be removed as soon as Solr is dropped
+if [[ "${FLATCURVE_EXPERIMENTAL}" =~ ^([yY][eE][sS]|[yY])$ ]]; then
+cat <<EOF > /etc/dovecot/conf.d/fts.conf
+# Autogenerated by mailcow
+plugin {
+    fts_autoindex = yes
+    fts_autoindex_exclude = \Junk
+    fts_autoindex_exclude2 = \Trash
+    fts = flatcurve
+
+    # These are not flatcurve settings, but required for Dovecot FTS. See
+    # Dovecot FTS Configuration link above for further information.
+    fts_languages = en es de
+    fts_tokenizer_generic = algorithm=simple
+    fts_tokenizers = generic email-address
+
+    # OPTIONAL: Recommended default FTS core configuration
+    fts_filters = normalizer-icu snowball stopwords
+    fts_filters_en = lowercase snowball english-possessive stopwords
+}
+EOF
+elif [[ ! "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])$ ]]; then
+cat <<EOF > /etc/dovecot/conf.d/fts.conf
+# Autogenerated by mailcow
+plugin {
+  fts = solr
+  fts_autoindex = yes
+  fts_autoindex_exclude = \Junk
+  fts_autoindex_exclude2 = \Trash
+  fts_solr = url=http://solr:8983/solr/dovecot-fts/
+
+  fts_tokenizers = generic email-address
+  fts_tokenizer_generic = algorithm=simple
+
+  fts_filters = normalizer-icu snowball stopwords
+  fts_filters_en = lowercase snowball english-possessive stopwords
+}
+EOF
+fi
+
+
 # Replace patterns in app-passdb.lua
 sed -i "s/__DBUSER__/${DBUSER}/g" /etc/dovecot/lua/passwd-verify.lua
 sed -i "s/__DBPASS__/${DBPASS}/g" /etc/dovecot/lua/passwd-verify.lua
@@ -343,7 +392,6 @@ mail_replica = tcp:${MAILCOW_REPLICA_IP}:${DOVEADM_REPLICA_PORT}
 EOF
 fi
 
-
 # 401 is user dovecot
 if [[ ! -s /mail_crypt/ecprivkey.pem || ! -s /mail_crypt/ecpubkey.pem ]]; then
 	openssl ecparam -name prime256v1 -genkey | openssl pkey -out /mail_crypt/ecprivkey.pem
@@ -387,7 +435,8 @@ chmod +x /usr/lib/dovecot/sieve/rspamd-pipe-ham \
   /usr/local/bin/maildir_gc.sh \
   /usr/local/sbin/stop-supervisor.sh \
   /usr/local/bin/quota_notify.py \
-  /usr/local/bin/repl_health.sh
+  /usr/local/bin/repl_health.sh \
+  /usr/local/bin/optimize-fts.sh
 
 # Prepare environment file for cronjobs
 printenv | sed 's/^\(.*\)$/export \1/g' > /source_env.sh

data/Dockerfiles/dovecot/optimize-fts.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [[ "${SKIP_SOLR}" =~ ^([yY][eE][sS]|[yY])+$ && ! "${FLATCURVE_EXPERIMENTAL}" =~ ^([yY][eE][sS]|[yY])+$ ]]; then
+    exit 0
+else
+    doveadm fts optimize -A
+fi

data/Dockerfiles/dovecot/sa-rules.sh

@@ -11,7 +11,7 @@ else
 fi
 
 # Deploy
-curl --connect-timeout 15 --retry 10 --max-time 30 http://www.spamassassin.heinlein-support.de/$(dig txt 1.4.3.spamassassin.heinlein-support.de +short | tr -d '"' | tr -dc '0-9').tar.gz --output /tmp/sa-rules-heinlein.tar.gz
+curl --connect-timeout 15 --retry 10 --max-time 30 https://www.spamassassin.heinlein-support.de/$(dig txt 1.4.3.spamassassin.heinlein-support.de +short | tr -d '"' | tr -dc '0-9').tar.gz --output /tmp/sa-rules-heinlein.tar.gz
 if gzip -t /tmp/sa-rules-heinlein.tar.gz; then
   tar xfvz /tmp/sa-rules-heinlein.tar.gz -C /tmp/sa-rules-heinlein
   cat /tmp/sa-rules-heinlein/*cf > /etc/rspamd/custom/sa-rules

data/Dockerfiles/netfilter/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.19
+FROM alpine:3.20
 LABEL maintainer "The Infrastructure Company GmbH <[email protected]>"
 
 WORKDIR /app

data/Dockerfiles/netfilter/main.py

@@ -80,16 +80,16 @@ def refreshF2bregex():
   global exit_code
   if not r.get('F2B_REGEX'):
     f2bregex = {}
-    f2bregex[1] = 'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)'
-    f2bregex[2] = 'Rspamd UI: Invalid password by ([0-9a-f\.:]+)'
-    f2bregex[3] = 'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+'
-    f2bregex[4] = 'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+'
-    f2bregex[5] = 'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+'
-    f2bregex[6] = '-login: Disconnected.+ \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),'
-    f2bregex[7] = '-login: Aborted login.+ \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
-    f2bregex[8] = '-login: Aborted login.+ \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
-    f2bregex[9] = 'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked'
-    f2bregex[10] = '([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+'
+    f2bregex[1] = r'mailcow UI: Invalid password for .+ by ([0-9a-f\.:]+)'
+    f2bregex[2] = r'Rspamd UI: Invalid password by ([0-9a-f\.:]+)'
+    f2bregex[3] = r'warning: .*\[([0-9a-f\.:]+)\]: SASL .+ authentication failed: (?!.*Connection lost to authentication server).+'
+    f2bregex[4] = r'warning: non-SMTP command from .*\[([0-9a-f\.:]+)]:.+'
+    f2bregex[5] = r'NOQUEUE: reject: RCPT from \[([0-9a-f\.:]+)].+Protocol error.+'
+    f2bregex[6] = r'-login: Disconnected.+ \(auth failed, .+\): user=.*, method=.+, rip=([0-9a-f\.:]+),'
+    f2bregex[7] = r'-login: Aborted login.+ \(auth failed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
+    f2bregex[8] = r'-login: Aborted login.+ \(tried to use disallowed .+\): user=.+, rip=([0-9a-f\.:]+), lip.+'
+    f2bregex[9] = r'SOGo.+ Login from \'([0-9a-f\.:]+)\' for user .+ might not have worked'
+    f2bregex[10] = r'([0-9a-f\.:]+) \"GET \/SOGo\/.* HTTP.+\" 403 .+'
     r.set('F2B_REGEX', json.dumps(f2bregex, ensure_ascii=False))
   else:
     try:

data/Dockerfiles/olefy/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.19
+FROM alpine:3.20
 LABEL maintainer "The Infrastructure Company GmbH <[email protected]>"
 
 ARG PIP_BREAK_SYSTEM_PACKAGES=1

data/Dockerfiles/phpfpm/Dockerfile

@@ -1,4 +1,4 @@
-FROM php:8.2-fpm-alpine3.18
+FROM php:8.2-fpm-alpine3.20
 LABEL maintainer "The Infrastructure Company GmbH <[email protected]>"
 
 # renovate: datasource=github-tags depName=krakjoe/apcu versioning=semver-coerced extractVersion=^v(?<version>.*)$

data/Dockerfiles/postfix/Dockerfile

@@ -1,4 +1,4 @@
-FROM debian:bullseye-slim
+FROM debian:bookworm-slim
 LABEL maintainer "The Infrastructure Company GmbH <[email protected]>"
 
 ARG DEBIAN_FRONTEND=noninteractive

data/Dockerfiles/postfix/postfix.sh

@@ -415,12 +415,6 @@ postscreen_dnsbl_sites = wl.mailspike.net=127.0.0.[18;19;20]*-2
   b.barracudacentral.org=127.0.0.2*7
   bl.mailspike.net=127.0.0.2*5
   bl.mailspike.net=127.0.0.[10;11;12]*4
-  dnsbl.sorbs.net=127.0.0.10*8
-  dnsbl.sorbs.net=127.0.0.5*6
-  dnsbl.sorbs.net=127.0.0.7*3
-  dnsbl.sorbs.net=127.0.0.8*2
-  dnsbl.sorbs.net=127.0.0.6*2
-  dnsbl.sorbs.net=127.0.0.9*2
 EOF
 fi
 DNSBL_CONFIG=$(grep -v '^#' /opt/postfix/conf/dns_blocklists.cf | grep '\S')

data/Dockerfiles/postfix/syslog-ng-redis_slave.conf

@@ -1,4 +1,4 @@
-@version: 3.28
+@version: 3.38
 @include "scl.conf"
 options {
   chain_hostnames(off);

data/Dockerfiles/postfix/syslog-ng.conf

@@ -1,4 +1,4 @@
-@version: 3.28
+@version: 3.38
 @include "scl.conf"
 options {
   chain_hostnames(off);

data/Dockerfiles/rspamd/Dockerfile

@@ -2,6 +2,7 @@ FROM debian:bullseye-slim
 LABEL maintainer "The Infrastructure Company GmbH <[email protected]>"
 
 ARG DEBIAN_FRONTEND=noninteractive
+ARG RSPAMD_VER=rspamd_3.7.5-2~8c86c1676   
 ARG CODENAME=bullseye
 ENV LC_ALL C
 
@@ -12,11 +13,14 @@ RUN apt-get update && apt-get install -y \
   apt-transport-https \
   dnsutils \
   netcat \
-  && apt-key adv --fetch-keys https://rspamd.com/apt-stable/gpg.key \
-  && echo "deb https://rspamd.com/apt-stable/ $CODENAME main" > /etc/apt/sources.list.d/rspamd.list \
-  && apt-get update \
-  && apt-get --no-install-recommends -y install rspamd redis-tools procps nano \
-  && rm -rf /var/lib/apt/lists/* \
+  wget \
+  redis-tools \ 
+  procps \ 
+  nano \
+  && arch=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) \
+  && wget -P /tmp https://rspamd.com/apt-stable/pool/main/r/rspamd/${RSPAMD_VER}~${CODENAME}_${arch}.deb\
+  && apt install -y /tmp/${RSPAMD_VER}~${CODENAME}_${arch}.deb \
+  && rm -rf /var/lib/apt/lists/* /tmp/*\
   && apt-get autoremove --purge \
   && apt-get clean \
   && mkdir -p /run/rspamd \
@@ -25,7 +29,6 @@ RUN apt-get update && apt-get install -y \
   && sed -i 's/#analysis_keyword_table > 0/analysis_cat_table.macro_exist == "M"/g' /usr/share/rspamd/lualib/lua_scanners/oletools.lua
 
 COPY settings.conf /etc/rspamd/settings.conf
-COPY metadata_exporter.lua /usr/share/rspamd/plugins/metadata_exporter.lua
 COPY set_worker_password.sh /set_worker_password.sh
 COPY docker-entrypoint.sh /docker-entrypoint.sh