feat: remove support for facebookexternalhit UA string

README.md

@@ -86,7 +86,7 @@ Please find a script named `fake-bot-report.sh` in the util folder.
 
 ## License
 
-Copyright (c) 2022-2024 OWASP CRS project. All rights reserved.
+Copyright (c) 2022-2025 OWASP CRS project. All rights reserved.
 
 The OWASP CRS and its official plugins are distributed
 under Apache Software License (ASL) version 2. Please see the enclosed LICENSE

plugins/fake-bot-after.conf

@@ -1,6 +1,6 @@
 # ------------------------------------------------------------------------
 # OWASP CRS Plugin
-# Copyright (c) 2022-2024 CRS project. All rights reserved.
+# Copyright (c) 2022-2025 CRS project. All rights reserved.
 #
 # The OWASP CRS plugins are distributed under
 # Apache Software License (ASL) version 2
@@ -29,7 +29,7 @@ SecRule TX:FAKE-BOT-PLUGIN_WHITELIST_BROKEN_APPLE_DEVICES "@streq 1" \
     chain"
     SecRule REQUEST_HEADERS:User-Agent "@endsWith facebookexternalhit/1.1 Facebot Twitterbot/1.0"
 
-SecRule REQUEST_HEADERS:User-Agent "@pm amazonbot applebot bingbot linkedinbot facebookbot facebookcatalog facebookexternalhit googlebot twitterbot" \
+SecRule REQUEST_HEADERS:User-Agent "@pm amazonbot applebot bingbot linkedinbot facebookbot facebookcatalog googlebot twitterbot" \
     "id:9504120,\
     phase:1,\
     block,\

plugins/fake-bot-before.conf

@@ -1,6 +1,6 @@
 # ------------------------------------------------------------------------
 # OWASP CRS Plugin
-# Copyright (c) 2022-2024 CRS project. All rights reserved.
+# Copyright (c) 2022-2025 CRS project. All rights reserved.
 #
 # The OWASP CRS plugins are distributed under
 # Apache Software License (ASL) version 2

plugins/fake-bot-config.conf

@@ -1,6 +1,6 @@
 # ------------------------------------------------------------------------
 # OWASP CRS Plugin
-# Copyright (c) 2022-2024 CRS project. All rights reserved.
+# Copyright (c) 2022-2025 CRS project. All rights reserved.
 #
 # The OWASP CRS plugins are distributed under
 # Apache Software License (ASL) version 2

plugins/fake-bot.lua

@@ -1,6 +1,6 @@
 -- -----------------------------------------------------------------------
 -- OWASP CRS Plugin
--- Copyright (c) 2022-2024 CRS project. All rights reserved.
+-- Copyright (c) 2022-2025 CRS project. All rights reserved.
 --
 -- The OWASP CRS plugins are distributed under
 -- Apache Software License (ASL) version 2
@@ -52,9 +52,9 @@ function main(matched_bot)
 		-- https://developers.google.com/search/docs/advanced/crawling/verifying-googlebot
 		bot_domains = {".googlebot.com", ".google.com"}
 		bot_name = "Googlebot"
-	elseif matched_bot == "facebookexternalhit" or matched_bot == "facebookcatalog" or matched_bot == "facebookbot" then
-		-- https://developers.facebook.com/docs/sharing/webmasters/crawler/
-		-- https://developers.facebook.com/docs/sharing/bot/
+	-- We can no longer support 'facebookexternalhit' UA string as Facebook started to use IP addresses without reverse record in DNS.
+	elseif matched_bot == "facebookcatalog" or matched_bot == "facebookbot" then
+		-- https://developers.facebook.com/docs/sharing/webmasters/web-crawlers
 		bot_domains = {".facebook.com", ".fbsv.net"}
 		bot_name = "Facebookbot"
 	elseif matched_bot == "bingbot" then

tests/regression/fake-bot-plugin/9504120.yaml

@@ -38,22 +38,6 @@ tests:
           output:
             log_contains: id "9504120"
   - test_title: 9504120-3
-    desc: Check for blocking of fake Facebookbot
-    stages:
-      - stage:
-          input:
-            dest_addr: 127.0.0.1
-            headers:
-              Host: localhost
-              User-Agent: "OWASP CRS test agent: facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)"
-              Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
-            port: 80
-            method: GET
-            version: HTTP/1.1
-            uri: /get
-          output:
-            log_contains: id "9504120"
-  - test_title: 9504120-4
     desc: Check for blocking of fake Bingbot
     stages:
       - stage:
@@ -69,7 +53,7 @@ tests:
             uri: /get
           output:
             log_contains: id "9504120"
-  - test_title: 9504120-5
+  - test_title: 9504120-4
     desc: Check for blocking of fake Twitterbot
     stages:
       - stage:
@@ -85,7 +69,7 @@ tests:
             uri: /get
           output:
             log_contains: id "9504120"
-  - test_title: 9504120-6
+  - test_title: 9504120-5
     desc: Check for blocking of fake Applebot
     stages:
       - stage:
@@ -101,7 +85,7 @@ tests:
             uri: /get
           output:
             log_contains: id "9504120"
-  - test_title: 9504120-7
+  - test_title: 9504120-6
     desc: Check for blocking of fake LinkedInBot
     stages:
       - stage:
@@ -117,7 +101,7 @@ tests:
             uri: /get
           output:
             log_contains: id "9504120"
-  - test_title: 9504120-8
+  - test_title: 9504120-7
     desc: Check for blocking of fake Amazonbot
     stages:
       - stage: