Merge pull request #5689 from mailcow/staging

2024-01c

.github/workflows/update_postscreen_access_list.yml

@@ -22,7 +22,7 @@ jobs:
  bash helper-scripts/update_postscreen_whitelist.sh
 
  - name: Create Pull Request
- uses: peter-evans/create-pull-request@v5
+ uses: peter-evans/create-pull-request@v6
  with:
  token: ${{ secrets.mailcow_action_Update_postscreen_access_cidr_pat }}
  commit-message: update postscreen_access.cidr

.gitignore

@@ -13,6 +13,7 @@ data/conf/dovecot/acl_anyone
 data/conf/dovecot/dovecot-master.passwd
 data/conf/dovecot/dovecot-master.userdb
 data/conf/dovecot/extra.conf
+data/conf/dovecot/mail_replica.conf
 data/conf/dovecot/global_sieve_*
 data/conf/dovecot/last_login
 data/conf/dovecot/lua

data/Dockerfiles/dovecot/docker-entrypoint.sh

@@ -335,6 +335,15 @@ sys.exit()
 EOF
 fi
 
+# Set mail_replica for HA setups
+if [[ -n ${MAILCOW_REPLICA_IP} && -n ${DOVEADM_REPLICA_PORT} ]]; then
+ cat <<EOF > /etc/dovecot/mail_replica.conf
+# Autogenerated by mailcow
+mail_replica = tcp:${MAILCOW_REPLICA_IP}:${DOVEADM_REPLICA_PORT}
+EOF
+fi
+
+
 # 401 is user dovecot
 if [[ ! -s /mail_crypt/ecprivkey.pem || ! -s /mail_crypt/ecpubkey.pem ]]; then
  openssl ecparam -name prime256v1 -genkey | openssl pkey -out /mail_crypt/ecprivkey.pem

data/Dockerfiles/netfilter/main.py

@@ -21,47 +21,17 @@
 from modules.NFTables import NFTables
 
 
-# connect to redis
-while True:
- try:
- redis_slaveof_ip = os.getenv('REDIS_SLAVEOF_IP', '')
- redis_slaveof_port = os.getenv('REDIS_SLAVEOF_PORT', '')
- if "".__eq__(redis_slaveof_ip):
- r = redis.StrictRedis(host=os.getenv('IPV4_NETWORK', '172.22.1') + '.249', decode_responses=True, port=6379, db=0)
- else:
- r = redis.StrictRedis(host=redis_slaveof_ip, decode_responses=True, port=redis_slaveof_port, db=0)
- r.ping()
- except Exception as ex:
- print('%s - trying again in 3 seconds' % (ex))
- time.sleep(3)
- else:
- break
-pubsub = r.pubsub()
-
-# rename fail2ban to netfilter
-if r.exists('F2B_LOG'):
- r.rename('F2B_LOG', 'NETFILTER_LOG')
-
-
 # globals
 WHITELIST = []
 BLACKLIST= []
 bans = {}
 quit_now = False
 exit_code = 0
 lock = Lock()
-
-
-# init Logger
-logger = Logger(r)
-# init backend
-backend = sys.argv[1]
-if backend == "nftables":
- logger.logInfo('Using NFTables backend')
- tables = NFTables("MAILCOW", logger)
-else:
- logger.logInfo('Using IPTables backend')
- tables = IPTables("MAILCOW", logger)
+chain_name = "MAILCOW"
+r = None
+pubsub = None
+clear_before_quit = False
 
 
 def refreshF2boptions():
@@ -250,17 +220,21 @@ def clear():
  with lock:
  tables.clearIPv4Table()
  tables.clearIPv6Table()
- r.delete('F2B_ACTIVE_BANS')
- r.delete('F2B_PERM_BANS')
- pubsub.unsubscribe()
+ try:
+ if r is not None:
+ r.delete('F2B_ACTIVE_BANS')
+ r.delete('F2B_PERM_BANS')
+ except Exception as ex:
+ logger.logWarn('Error clearing redis keys F2B_ACTIVE_BANS and F2B_PERM_BANS: %s' % ex)
 
 def watch():
- logger.logInfo('Watching Redis channel F2B_CHANNEL')
- pubsub.subscribe('F2B_CHANNEL')
-
+ global pubsub
  global quit_now
  global exit_code
 
+ logger.logInfo('Watching Redis channel F2B_CHANNEL')
+ pubsub.subscribe('F2B_CHANNEL')
+
  while not quit_now:
  try:
  for item in pubsub.listen():
@@ -280,6 +254,7 @@ def watch():
  ban(addr)
  except Exception as ex:
  logger.logWarn('Error reading log line from pubsub: %s' % ex)
+ pubsub = None
  quit_now = True
  exit_code = 2
 
@@ -403,21 +378,76 @@ def blacklistUpdate():
  permBan(net=net, unban=True)
  time.sleep(60.0 - ((time.time() - start_time) % 60.0))
 
-def quit(signum, frame):
- global quit_now
- quit_now = True
+def sigterm_quit(signum, frame):
+ global clear_before_quit
+ clear_before_quit = True
+ sys.exit(exit_code)
+
+def berfore_quit():
+ if clear_before_quit:
+ clear()
+ if pubsub is not None:
+ pubsub.unsubscribe()
 
 
 if __name__ == '__main__':
- refreshF2boptions()
+ atexit.register(berfore_quit)
+ signal.signal(signal.SIGTERM, sigterm_quit)
+
+ # init Logger
+ logger = Logger(None)
+
+ # init backend
+ backend = sys.argv[1]
+ if backend == "nftables":
+ logger.logInfo('Using NFTables backend')
+ tables = NFTables(chain_name, logger)
+ else:
+ logger.logInfo('Using IPTables backend')
+ tables = IPTables(chain_name, logger)
+
  # In case a previous session was killed without cleanup
  clear()
+
  # Reinit MAILCOW chain
  # Is called before threads start, no locking
  logger.logInfo("Initializing mailcow netfilter chain")
  tables.initChainIPv4()
  tables.initChainIPv6()
 
+ if os.getenv("DISABLE_NETFILTER_ISOLATION_RULE").lower() in ("y", "yes"):
+ logger.logInfo(f"Skipping {chain_name} isolation")
+ else:
+ logger.logInfo(f"Setting {chain_name} isolation")
+ tables.create_mailcow_isolation_rule("br-mailcow", [3306, 6379, 8983, 12345], os.getenv("MAILCOW_REPLICA_IP"))
+
+ # connect to redis
+ while True:
+ try:
+ redis_slaveof_ip = os.getenv('REDIS_SLAVEOF_IP', '')
+ redis_slaveof_port = os.getenv('REDIS_SLAVEOF_PORT', '')
+ if "".__eq__(redis_slaveof_ip):
+ r = redis.StrictRedis(host=os.getenv('IPV4_NETWORK', '172.22.1') + '.249', decode_responses=True, port=6379, db=0)
+ else:
+ r = redis.StrictRedis(host=redis_slaveof_ip, decode_responses=True, port=redis_slaveof_port, db=0)
+ r.ping()
+ pubsub = r.pubsub()
+ except Exception as ex:
+ print('%s - trying again in 3 seconds' % (ex))
+ time.sleep(3)
+ else:
+ break
+ Logger.r = r
+
+ # rename fail2ban to netfilter
+ if r.exists('F2B_LOG'):
+ r.rename('F2B_LOG', 'NETFILTER_LOG')
+ # clear bans in redis
+ r.delete('F2B_ACTIVE_BANS')
+ r.delete('F2B_PERM_BANS')
+
+ refreshF2boptions()
+
  watch_thread = Thread(target=watch)
  watch_thread.daemon = True
  watch_thread.start()
@@ -460,9 +490,6 @@ def quit(signum, frame):
  whitelistupdate_thread.daemon = True
  whitelistupdate_thread.start()
 
- signal.signal(signal.SIGTERM, quit)
- atexit.register(clear)
-
  while not quit_now:
  time.sleep(0.5)
 

data/Dockerfiles/netfilter/modules/IPTables.py

@@ -1,5 +1,6 @@
 import iptc
 import time
+import os
 
 class IPTables:
  def __init__(self, chain_name, logger):
@@ -211,3 +212,41 @@ def getSnat6Rule(self, snat_target, source):
  target = rule.create_target("SNAT")
  target.to_source = snat_target
  return rule
+
+ def create_mailcow_isolation_rule(self, _interface:str, _dports:list, _allow:str = ""):
+ try:
+ chain = iptc.Chain(iptc.Table(iptc.Table.FILTER), self.chain_name)
+
+ # insert mailcow isolation rule
+ rule = iptc.Rule()
+ rule.in_interface = f'! {_interface}'
+ rule.out_interface = _interface
+ rule.protocol = 'tcp'
+ rule.create_target("DROP")
+ match = rule.create_match("multiport")
+ match.dports = ','.join(map(str, _dports))
+
+ if rule in chain.rules:
+ chain.delete_rule(rule)
+ chain.insert_rule(rule, position=0)
+
+ # insert mailcow isolation exception rule
+ if _allow != "":
+ rule = iptc.Rule()
+ rule.src = _allow
+ rule.in_interface = f'! {_interface}'
+ rule.out_interface = _interface
+ rule.protocol = 'tcp'
+ rule.create_target("ACCEPT")
+ match = rule.create_match("multiport")
+ match.dports = ','.join(map(str, _dports))
+
+ if rule in chain.rules:
+ chain.delete_rule(rule)
+ chain.insert_rule(rule, position=0)
+
+
+ return True
+ except Exception as e:
+ self.logger.logCrit(f"Error adding {self.chain_name} isolation: {e}")
+ return False

data/Dockerfiles/netfilter/modules/Logger.py

@@ -10,7 +10,8 @@ def log(self, priority, message):
  tolog['time'] = int(round(time.time()))
  tolog['priority'] = priority
  tolog['message'] = message
- self.r.lpush('NETFILTER_LOG', json.dumps(tolog, ensure_ascii=False))
+ if self.r:
+ self.r.lpush('NETFILTER_LOG', json.dumps(tolog, ensure_ascii=False))
  print(message)
 
  def logWarn(self, message):