From 5cb2182738b6683f4b2c84c51fac490e9d4495d2 Mon Sep 17 00:00:00 2001 From: azurit Date: Fri, 17 Jan 2025 15:20:55 +0100 Subject: [PATCH 1/9] Update fake-bot-after.conf --- plugins/fake-bot-after.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/fake-bot-after.conf b/plugins/fake-bot-after.conf index 89deed9..4c9f873 100644 --- a/plugins/fake-bot-after.conf +++ b/plugins/fake-bot-after.conf @@ -29,7 +29,7 @@ SecRule TX:FAKE-BOT-PLUGIN_WHITELIST_BROKEN_APPLE_DEVICES "@streq 1" \ chain" SecRule REQUEST_HEADERS:User-Agent "@endsWith facebookexternalhit/1.1 Facebot Twitterbot/1.0" -SecRule REQUEST_HEADERS:User-Agent "@pm amazonbot applebot bingbot linkedinbot facebookbot facebookcatalog facebookexternalhit googlebot twitterbot" \ +SecRule REQUEST_HEADERS:User-Agent "@pm amazonbot applebot bingbot linkedinbot facebookbot facebookcatalog googlebot twitterbot" \ "id:9504120,\ phase:1,\ block,\ From 084f94c01f2f24aa80870dce2f7a713033b180c9 Mon Sep 17 00:00:00 2001 From: azurit Date: Fri, 17 Jan 2025 15:22:31 +0100 Subject: [PATCH 2/9] Update fake-bot.lua --- plugins/fake-bot.lua | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/fake-bot.lua b/plugins/fake-bot.lua index 37b1d6c..786fba3 100644 --- a/plugins/fake-bot.lua +++ b/plugins/fake-bot.lua @@ -52,9 +52,9 @@ function main(matched_bot) -- https://developers.google.com/search/docs/advanced/crawling/verifying-googlebot bot_domains = {".googlebot.com", ".google.com"} bot_name = "Googlebot" - elseif matched_bot == "facebookexternalhit" or matched_bot == "facebookcatalog" or matched_bot == "facebookbot" then - -- https://developers.facebook.com/docs/sharing/webmasters/crawler/ - -- https://developers.facebook.com/docs/sharing/bot/ + -- We can no longer support 'facebookexternalhit' UA string as Facebook started to use IP addresses without reverse record in DNS. + elseif matched_bot == "facebookcatalog" or matched_bot == "facebookbot" then + -- https://developers.facebook.com/docs/sharing/webmasters/web-crawlers bot_domains = {".facebook.com", ".fbsv.net"} bot_name = "Facebookbot" elseif matched_bot == "bingbot" then From a532acd80c2cbb549bc75e9ebccb16456ae9152e Mon Sep 17 00:00:00 2001 From: azurit Date: Fri, 17 Jan 2025 15:22:57 +0100 Subject: [PATCH 3/9] Update fake-bot-after.conf --- plugins/fake-bot-after.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/fake-bot-after.conf b/plugins/fake-bot-after.conf index 4c9f873..50f799e 100644 --- a/plugins/fake-bot-after.conf +++ b/plugins/fake-bot-after.conf @@ -1,6 +1,6 @@ # ------------------------------------------------------------------------ # OWASP CRS Plugin -# Copyright (c) 2022-2024 CRS project. All rights reserved. +# Copyright (c) 2022-2025 CRS project. All rights reserved. # # The OWASP CRS plugins are distributed under # Apache Software License (ASL) version 2 From 979c031ae0ccd5cb7d773b66167b9ad19150b2de Mon Sep 17 00:00:00 2001 From: azurit Date: Fri, 17 Jan 2025 15:23:13 +0100 Subject: [PATCH 4/9] Update fake-bot.lua --- plugins/fake-bot.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/fake-bot.lua b/plugins/fake-bot.lua index 786fba3..7d1efd9 100644 --- a/plugins/fake-bot.lua +++ b/plugins/fake-bot.lua @@ -1,6 +1,6 @@ -- ----------------------------------------------------------------------- -- OWASP CRS Plugin --- Copyright (c) 2022-2024 CRS project. All rights reserved. +-- Copyright (c) 2022-2025 CRS project. All rights reserved. -- -- The OWASP CRS plugins are distributed under -- Apache Software License (ASL) version 2 From 5ef023b5034cb59383441f49ff99532268461776 Mon Sep 17 00:00:00 2001 From: azurit Date: Fri, 17 Jan 2025 15:23:24 +0100 Subject: [PATCH 5/9] Update fake-bot-before.conf --- plugins/fake-bot-before.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/fake-bot-before.conf b/plugins/fake-bot-before.conf index 227bec2..d8c9c67 100644 --- a/plugins/fake-bot-before.conf +++ b/plugins/fake-bot-before.conf @@ -1,6 +1,6 @@ # ------------------------------------------------------------------------ # OWASP CRS Plugin -# Copyright (c) 2022-2024 CRS project. All rights reserved. +# Copyright (c) 2022-2025 CRS project. All rights reserved. # # The OWASP CRS plugins are distributed under # Apache Software License (ASL) version 2 From c3abc5c63eb00eaf39479b521b8d5579842ddd2d Mon Sep 17 00:00:00 2001 From: azurit Date: Fri, 17 Jan 2025 15:23:35 +0100 Subject: [PATCH 6/9] Update fake-bot-config.conf --- plugins/fake-bot-config.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/fake-bot-config.conf b/plugins/fake-bot-config.conf index f9b40cd..6073283 100644 --- a/plugins/fake-bot-config.conf +++ b/plugins/fake-bot-config.conf @@ -1,6 +1,6 @@ # ------------------------------------------------------------------------ # OWASP CRS Plugin -# Copyright (c) 2022-2024 CRS project. All rights reserved. +# Copyright (c) 2022-2025 CRS project. All rights reserved. # # The OWASP CRS plugins are distributed under # Apache Software License (ASL) version 2 From ceb47f8fbb8a6d7d46af0a2cb4ee0aa9cf396096 Mon Sep 17 00:00:00 2001 From: azurit Date: Fri, 17 Jan 2025 15:23:58 +0100 Subject: [PATCH 7/9] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index aa04fd5..5c21333 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ Please find a script named `fake-bot-report.sh` in the util folder. ## License -Copyright (c) 2022-2024 OWASP CRS project. All rights reserved. +Copyright (c) 2022-2025 OWASP CRS project. All rights reserved. The OWASP CRS and its official plugins are distributed under Apache Software License (ASL) version 2. Please see the enclosed LICENSE From f10983d6a0137a0923040fcd93833e3a92b84797 Mon Sep 17 00:00:00 2001 From: azurit Date: Fri, 17 Jan 2025 19:16:52 +0100 Subject: [PATCH 8/9] Update 9504120.yaml --- tests/regression/fake-bot-plugin/9504120.yaml | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/tests/regression/fake-bot-plugin/9504120.yaml b/tests/regression/fake-bot-plugin/9504120.yaml index 3089b4d..e3d2c7e 100644 --- a/tests/regression/fake-bot-plugin/9504120.yaml +++ b/tests/regression/fake-bot-plugin/9504120.yaml @@ -37,22 +37,22 @@ tests: uri: /get output: log_contains: id "9504120" - - test_title: 9504120-3 - desc: Check for blocking of fake Facebookbot - stages: - - stage: - input: - dest_addr: 127.0.0.1 - headers: - Host: localhost - User-Agent: "OWASP CRS test agent: facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)" - Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5 - port: 80 - method: GET - version: HTTP/1.1 - uri: /get - output: - log_contains: id "9504120" +# - test_title: 9504120-3 +# desc: Check for blocking of fake Facebookbot +# stages: +# - stage: +# input: +# dest_addr: 127.0.0.1 +# headers: +# Host: localhost +# User-Agent: "OWASP CRS test agent: facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)" +# Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5 +# port: 80 +# method: GET +# version: HTTP/1.1 +# uri: /get +# output: +# log_contains: id "9504120" - test_title: 9504120-4 desc: Check for blocking of fake Bingbot stages: From 579809eb39f469baa5ec1fa677f3833e965d9789 Mon Sep 17 00:00:00 2001 From: azurit Date: Fri, 17 Jan 2025 19:58:33 +0100 Subject: [PATCH 9/9] Update 9504120.yaml --- tests/regression/fake-bot-plugin/9504120.yaml | 26 ++++--------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/tests/regression/fake-bot-plugin/9504120.yaml b/tests/regression/fake-bot-plugin/9504120.yaml index e3d2c7e..f0e8ae4 100644 --- a/tests/regression/fake-bot-plugin/9504120.yaml +++ b/tests/regression/fake-bot-plugin/9504120.yaml @@ -37,23 +37,7 @@ tests: uri: /get output: log_contains: id "9504120" -# - test_title: 9504120-3 -# desc: Check for blocking of fake Facebookbot -# stages: -# - stage: -# input: -# dest_addr: 127.0.0.1 -# headers: -# Host: localhost -# User-Agent: "OWASP CRS test agent: facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)" -# Accept: text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5 -# port: 80 -# method: GET -# version: HTTP/1.1 -# uri: /get -# output: -# log_contains: id "9504120" - - test_title: 9504120-4 + - test_title: 9504120-3 desc: Check for blocking of fake Bingbot stages: - stage: @@ -69,7 +53,7 @@ tests: uri: /get output: log_contains: id "9504120" - - test_title: 9504120-5 + - test_title: 9504120-4 desc: Check for blocking of fake Twitterbot stages: - stage: @@ -85,7 +69,7 @@ tests: uri: /get output: log_contains: id "9504120" - - test_title: 9504120-6 + - test_title: 9504120-5 desc: Check for blocking of fake Applebot stages: - stage: @@ -101,7 +85,7 @@ tests: uri: /get output: log_contains: id "9504120" - - test_title: 9504120-7 + - test_title: 9504120-6 desc: Check for blocking of fake LinkedInBot stages: - stage: @@ -117,7 +101,7 @@ tests: uri: /get output: log_contains: id "9504120" - - test_title: 9504120-8 + - test_title: 9504120-7 desc: Check for blocking of fake Amazonbot stages: - stage: