Merge pull request #24 from azurit/facebookexternalhit

feat: remove support for facebookexternalhit UA string
coreruleset · Jan 18, 2025 · d8be69e · d8be69e
2 parents 6177599 + 579809e
commit d8be69e
Show file tree

Hide file tree

Showing 6 changed files with 13 additions and 29 deletions.
diff --git a/README.md b/README.md
@@ -86,7 +86,7 @@ Please find a script named `fake-bot-report.sh` in the util folder.
 
 ## License
 
-Copyright (c) 2022-2024 OWASP CRS project. All rights reserved.
+Copyright (c) 2022-2025 OWASP CRS project. All rights reserved.
 
 The OWASP CRS and its official plugins are distributed
 under Apache Software License (ASL) version 2. Please see the enclosed LICENSE

diff --git a/plugins/fake-bot-after.conf b/plugins/fake-bot-after.conf
@@ -1,6 +1,6 @@
 # ------------------------------------------------------------------------
 # OWASP CRS Plugin
-# Copyright (c) 2022-2024 CRS project. All rights reserved.
+# Copyright (c) 2022-2025 CRS project. All rights reserved.
 #
 # The OWASP CRS plugins are distributed under
 # Apache Software License (ASL) version 2
@@ -29,7 +29,7 @@ SecRule TX:FAKE-BOT-PLUGIN_WHITELIST_BROKEN_APPLE_DEVICES "@streq 1" \
     chain"
     SecRule REQUEST_HEADERS:User-Agent "@endsWith facebookexternalhit/1.1 Facebot Twitterbot/1.0"
 
-SecRule REQUEST_HEADERS:User-Agent "@pm amazonbot applebot bingbot linkedinbot facebookbot facebookcatalog facebookexternalhit googlebot twitterbot" \
+SecRule REQUEST_HEADERS:User-Agent "@pm amazonbot applebot bingbot linkedinbot facebookbot facebookcatalog googlebot twitterbot" \
     "id:9504120,\
     phase:1,\
     block,\

diff --git a/plugins/fake-bot-before.conf b/plugins/fake-bot-before.conf
@@ -1,6 +1,6 @@
 # ------------------------------------------------------------------------
 # OWASP CRS Plugin
-# Copyright (c) 2022-2024 CRS project. All rights reserved.
+# Copyright (c) 2022-2025 CRS project. All rights reserved.
 #
 # The OWASP CRS plugins are distributed under
 # Apache Software License (ASL) version 2

diff --git a/plugins/fake-bot-config.conf b/plugins/fake-bot-config.conf
@@ -1,6 +1,6 @@
 # ------------------------------------------------------------------------
 # OWASP CRS Plugin
-# Copyright (c) 2022-2024 CRS project. All rights reserved.
+# Copyright (c) 2022-2025 CRS project. All rights reserved.
 #
 # The OWASP CRS plugins are distributed under
 # Apache Software License (ASL) version 2

diff --git a/plugins/fake-bot.lua b/plugins/fake-bot.lua
@@ -1,6 +1,6 @@
 -- -----------------------------------------------------------------------
 -- OWASP CRS Plugin
--- Copyright (c) 2022-2024 CRS project. All rights reserved.
+-- Copyright (c) 2022-2025 CRS project. All rights reserved.
 --
 -- The OWASP CRS plugins are distributed under
 -- Apache Software License (ASL) version 2
@@ -52,9 +52,9 @@ function main(matched_bot)
 		-- https://developers.google.com/search/docs/advanced/crawling/verifying-googlebot
 		bot_domains = {".googlebot.com", ".google.com"}
 		bot_name = "Googlebot"
-	elseif matched_bot == "facebookexternalhit" or matched_bot == "facebookcatalog" or matched_bot == "facebookbot" then
-		-- https://developers.facebook.com/docs/sharing/webmasters/crawler/
-		-- https://developers.facebook.com/docs/sharing/bot/
+	-- We can no longer support 'facebookexternalhit' UA string as Facebook started to use IP addresses without reverse record in DNS.
+	elseif matched_bot == "facebookcatalog" or matched_bot == "facebookbot" then
+		-- https://developers.facebook.com/docs/sharing/webmasters/web-crawlers
 		bot_domains = {".facebook.com", ".fbsv.net"}
 		bot_name = "Facebookbot"
 	elseif matched_bot == "bingbot" then

diff --git a/tests/regression/fake-bot-plugin/9504120.yaml b/tests/regression/fake-bot-plugin/9504120.yaml
@@ -38,22 +38,6 @@ tests:
           output:
             log_contains: id "9504120"
   - test_title: 9504120-3
-    desc: Check for blocking of fake Facebookbot
-    stages:
-      - stage:
-          input:
-            dest_addr: 127.0.0.1
-            headers:
-              Host: localhost
-              User-Agent: "OWASP CRS test agent: facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)"
-              Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5
-            port: 80
-            method: GET
-            version: HTTP/1.1
-            uri: /get
-          output:
-            log_contains: id "9504120"
-  - test_title: 9504120-4
     desc: Check for blocking of fake Bingbot
     stages:
       - stage:
@@ -69,7 +53,7 @@ tests:
             uri: /get
           output:
             log_contains: id "9504120"
-  - test_title: 9504120-5
+  - test_title: 9504120-4
     desc: Check for blocking of fake Twitterbot
     stages:
       - stage:
@@ -85,7 +69,7 @@ tests:
             uri: /get
           output:
             log_contains: id "9504120"
-  - test_title: 9504120-6
+  - test_title: 9504120-5
     desc: Check for blocking of fake Applebot
     stages:
       - stage:
@@ -101,7 +85,7 @@ tests:
             uri: /get
           output:
             log_contains: id "9504120"
-  - test_title: 9504120-7
+  - test_title: 9504120-6
     desc: Check for blocking of fake LinkedInBot
     stages:
       - stage:
@@ -117,7 +101,7 @@ tests:
             uri: /get
           output:
             log_contains: id "9504120"
-  - test_title: 9504120-8
+  - test_title: 9504120-7
     desc: Check for blocking of fake Amazonbot
     stages:
       - stage: