Merge pull request fail2ban#3701 from bes-internal/exim

filter.d/exim.conf: rewrite host line regex for all varied exim's log_selector states

ChangeLog

@@ -28,9 +28,11 @@ ver. 1.0.3-dev-1 (20??/??/??) - development nightly edition
   (value read from `/proc/sys/net/ipv6/conf/all/disable_ipv6`) if available, otherwise seeks over local IPv6 from network interfaces
   if available for platform and uses DNS to find local IPv6 as a fallback only
 * improve `ignoreself` by considering all local addresses from network interfaces additionally to IPs from hostnames (gh-3132)
+* `filter.d/exim.conf`:
+  - rewrite host line regex for all varied exim's log_selector states (gh-3263)
+  - fixed "dropped: too many ..." regex, also matching unrecognized commands now (gh-3502)
 * `action.d/mikrotik.conf` - new action for mikrotik routerOS, adds and removes entries from address lists on the router (gh-2860)
 * `action.d/smtp.py` - added optional support for TLS connections via the `ssl` arg.
-* `filter.d/exim.conf` - fixed "dropped: too many ..." regex, also matching unrecognized commands now (gh-3502)
 * `filter.d/nginx-forbidden.conf` - new filter to ban forbidden locations, e. g. using `deny` directive (gh-2226)
 * `filter.d/sshd.conf`:
   - avoid double counting for "maximum authentication attempts exceeded" (gh-3502)

config/filter.d/exim-common.conf

@@ -9,12 +9,33 @@ after = exim-common.local
 
 [Definition]
 
-host_info_pre = (?:H=([\w.-]+ )?(?:\(\S+\) )?)?
-host_info_suf = (?::\d+)?(?: I=\[\S+\](:\d+)?)?(?: U=\S+)?(?: P=e?smtp)?(?: F=(?:<>|[^@]+@\S+))?\s
-host_info = %(host_info_pre)s\[<HOST>\]%(host_info_suf)s
+_fields_grp = (?: (?!H=)[A-Za-z]{1,4}(?:=\S+)?)*
+host_info = %(_fields_grp)s (?:H=)?(?:[\w.-]+)? ?(?:\(\S+\))? ?\[<ADDR>\](?::\d+)?%(_fields_grp)s
 pid = (?: \[\d+\]| \w+ exim\[\d+\]:)?
 
-# DEV Notes:
-# From exim source code: ./src/receive.c:add_host_info_for_log
-#
-# Author:  Daniel Black
+
+# DEV Notes
+# ------------
+# Host string happens:
+# H=[ip address]
+# H=(helo_name) [ip address]
+# H=host_name [ip address]
+# H=host_name (helo_name) [ip address]
+# flags H=host_name (helo_name) [ip address] flags
+# where only [ip address] always visible, ignore ident
+# From exim source code:
+#   src/src/host.c:host_and_ident()
+#   src/receive.c:add_host_info_for_log()
+
+# Substitution of `_fields_grp` bypasses all flags but H
+# Summary of Fields in Log Lines depending on log_selector
+# https://www.exim.org/exim-html-current/doc/html/spec_html/ch-log_files.html
+# at version exim-4.97.1
+# ---
+
+# Authors:
+#   Cyril Jaquier
+#   Daniel Black (rewrote with strong regexs)
+#   Sergey G. Brester aka sebres (optimization, rewrite to prefregex, reviews)
+#   Martin O'Neal (added additional regexs to detect authentication failures, protocol errors, and drops)
+#   Varlamov Vladimir (host line definition)

config/filter.d/exim-spam.conf

@@ -26,9 +26,9 @@ before = exim-common.conf
 
 [Definition]
 
-failregex =  ^%(pid)s \S+ F=(<>|\S+@\S+) %(host_info)srejected by local_scan\(\): .{0,256}$
-             ^%(pid)s %(host_info)sF=(<>|[^@]+@\S+) rejected RCPT [^@]+@\S+: .*dnsbl.*\s*$
-             ^%(pid)s \S+ %(host_info)sF=(<>|[^@]+@\S+) rejected after DATA: This message contains a virus \(\S+\)\.\s*$
+failregex =  ^%(pid)s \S+%(host_info)s rejected by local_scan\(\): .{0,256}$
+             ^%(pid)s%(host_info)s rejected RCPT [^@]+@\S+: .*dnsbl.*\s*$
+             ^%(pid)s \S+%(host_info)s rejected after DATA: This message contains a virus \(\S+\)\.\s*$
              ^%(pid)s \S+ SA: Action: flagged as Spam but accepted: score=\d+\.\d+ required=\d+\.\d+ \(scanned in \d+/\d+ secs \| Message-Id: \S+\)\. From \S+ \(host=\S+ \[<HOST>\]\) for <honeypot>$
              ^%(pid)s \S+ SA: Action: silently tossed message: score=\d+\.\d+ required=\d+\.\d+ trigger=\d+\.\d+ \(scanned in \d+/\d+ secs \| Message-Id: \S+\)\. From \S+ \(host=(\S+ )?\[<HOST>\]\) for \S+$
 
@@ -43,8 +43,6 @@ ignoreregex =
 
 honeypot = [email protected]
 
-# DEV Notes:
-# The %(host_info) definition contains a <HOST> match
-#
-# Author: Cyril Jaquier
-#         Daniel Black (rewrote with strong regexs)
+# DEV Notes
+# -----------
+# The %(host_info) definition contains a <HOST> match. No space before. See exim-common.conf

config/filter.d/exim.conf

@@ -14,16 +14,16 @@ before = exim-common.conf
 [Definition]
 
 # Fre-filter via "prefregex" is currently inactive because of too different failure syntax in exim-log (testing needed):
-#prefregex = ^%(pid)s <F-CONTENT>\b(?:\w+ authenticator failed|([\w\-]+ )?SMTP (?:(?:call|connection) from|protocol(?: synchronization)? error)|no MAIL in|(?:%(host_info_pre)s\[[^\]]+\]%(host_info_suf)s(?:sender verify fail|rejected RCPT|dropped|AUTH command))).+</F-CONTENT>$
-
-failregex = ^%(pid)s %(host_info)ssender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$
-            ^%(pid)s \w+ authenticator failed for (?:[^\[\( ]* )?(?:\(\S*\) )?\[<HOST>\](?::\d+)?(?: I=\[\S+\](:\d+)?)?: 535 Incorrect authentication data( \(set_id=.*\)|: \d+ Time\(s\))?\s*$
-            ^%(pid)s %(host_info)srejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user|Unrouteable address)\s*$
-            ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+") %(host_info)s(?:next )?input=".*"\s*$
-            ^%(pid)s SMTP call from (?:[^\[\( ]* )?%(host_info)sdropped: too many (?:(?:nonmail|unrecognized) commands|syntax or protocol errors)
-            ^%(pid)s SMTP protocol error in "[^"]+(?:"+[^"]*(?="))*?" %(host_info)sAUTH command used when not advertised\s*$
-            ^%(pid)s no MAIL in SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sD=\d\S*s(?: C=\S*)?\s*$
-            ^%(pid)s (?:[\w\-]+ )?SMTP connection from (?:[^\[\( ]* )?(?:\(\S*\) )?%(host_info)sclosed by DROP in ACL\s*$
+#prefregex = ^%(pid)s <F-CONTENT>\b(?:\w+ authenticator failed|([\w\-]+ )?SMTP (?:(?:call|connection) from|protocol(?: synchronization)? error)|no MAIL in|(?:%(host_info)s(?:sender verify fail|rejected RCPT|dropped|AUTH command))).+</F-CONTENT>$
+
+failregex = ^%(pid)s%(host_info)s sender verify fail for <\S+>: (?:Unknown user|Unrouteable address|all relevant MX records point to non-existent hosts)\s*$
+            ^%(pid)s \w+ authenticator failed for%(host_info)s: 535 Incorrect authentication data(?: \(set_id=.*\)|: \d+ Time\(s\))?\s*$
+            ^%(pid)s%(host_info)s rejected RCPT [^@]+@\S+: (?:relay not permitted|Sender verify failed|Unknown user|Unrouteable address)\s*$
+            ^%(pid)s SMTP protocol synchronization error \([^)]*\): rejected (?:connection from|"\S+")%(host_info)s (?:next )?input=".*"\s*$
+            ^%(pid)s SMTP call from%(host_info)s dropped: too many (?:(?:nonmail|unrecognized) commands|syntax or protocol errors)
+            ^%(pid)s SMTP protocol error in "[^"]+(?:"+[^"]*(?="))*?"%(host_info)s AUTH command used when not advertised\s*$
+            ^%(pid)s no MAIL in SMTP connection from%(host_info)s
+            ^%(pid)s (?:[\w\-]+ )?SMTP connection from%(host_info)s closed by DROP in ACL\s*$
             <mdre-<mode>>
 
 mdre-aggressive = ^%(pid)s no host name found for IP address <HOST>$
@@ -42,13 +42,10 @@ mode = normal
 
 ignoreregex = 
 
-# DEV Notes:
-# The %(host_info) definition contains a <HOST> match
+# DEV Notes
+# -----------
+# The %(host_info) definition contains a <HOST> match. No space before. See exim-common.conf
 #
 # SMTP protocol synchronization error \([^)]*\)  <- This needs to be non-greedy
 # to void capture beyond ")" to avoid a DoS Injection vulnerability as input= is
 # user injectable data.
-#
-# Author: Cyril Jaquier
-#         Daniel Black (rewrote with strong regexs)
-#         Martin O'Neal (added additional regexs to detect authentication failures, protocol errors, and drops)

fail2ban/tests/files/logs/exim

@@ -106,3 +106,8 @@
 2017-12-03 08:32:00 no host name found for IP address 192.0.2.8
 # failJSON: { "time": "2017-12-03T08:51:35", "match": true , "host": "192.0.2.9", "desc": "no IP found for host" }
 2017-12-03 08:51:35 no IP address found for host test.example.com (during SMTP connection from [192.0.2.9])
+
+# failJSON: { "time": "2024-03-21T19:26:06", "match": true , "host": "194.169.175.1" }
+2024-03-21 19:26:06 dovecot_login authenticator failed for (User) [194.169.175.1]:21298 I=[22.33.44.55]:465 Ci=30416: 535 Incorrect authentication data ([email protected])
+# failJSON: { "time": "2024-03-21T09:18:51", "match": true , "host": "9.12.1.21" }
+2024-03-21 09:18:51 H=m05.horp.tld [9.12.1.21]:43030 I=[194.169.175.2]:25 Ci=7326 CV=no SNI=mail.leone.tld F=<[email protected]> rejected RCPT <[email protected]>: relay not permitted