From 07b065d6228083fd4a289eae69bf84f904a66308 Mon Sep 17 00:00:00 2001 From: mitchellkrogza Date: Wed, 24 Apr 2024 13:01:04 +0000 Subject: [PATCH] V4.2024.04.4457 [ci skip] --- AUTO-CONFIGURATION.md | 2 +- MANUAL-CONFIGURATION.md | 2 +- README.md | 2 +- _google_analytics_ghost_spam/README.md | 2 +- .../etc/nginx/conf.d/globalblacklist.conf | 5 +- conf.d/globalblacklist-testing-version.conf | 5 +- conf.d/globalblacklist.conf | 5 +- dev-tools/buildnumber | 2 +- dev-tools/globalblacklist-testing.template | 4 +- dev-tools/globalblacklist.template | 4 +- .../bots.d/blacklist-user-agents.conf | 25 ++- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 5 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 5 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 5 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 5 +- .../bots.d/blacklist-user-agents.conf | 25 ++- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 5 +- .../bots.d/blacklist-user-agents.conf | 25 ++- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 5 +- dev-tools/test_units/blacklist-ips.conf | 2 +- .../random-bots-for-test-quick.list | 18 +- .../random-bots-for-whitelist-test.list | 160 +++++++------- .../random-referrers-for-test-quick.list | 20 +- .../random-referrers-for-whitelist-test.list | 196 +++++++++--------- dev-tools/test_units/ratelimittest-master.txt | 22 +- dev-tools/test_units/whitelist-ips.conf | 2 +- dev-tools/tmp/robots.txt | 4 +- robots.txt/robots.txt | 4 +- 37 files changed, 319 insertions(+), 265 deletions(-) diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 8ee17d531..d7776c8e7 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2024.04.4456 +#### Version: V4.2024.04.4457 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [659](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 15fe86d05..777f8dac2 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2024.04.4456 +#### Version: V4.2024.04.4457 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [659](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/README.md b/README.md index c993af99e..411ec1383 100755 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2024.04.4456 +#### Version: V4.2024.04.4457 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [659](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index fb30a0fc2..48c56afb9 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,7 +3,7 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2024.04.4456 +#### Version: V4.2024.04.4457 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [659](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index c0a378cc6..39719fcf7 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.04.4456 -### Updated: Wed Apr 24 12:42:07 UTC 2024 +### Version: V4.2024.04.4457 +### Updated: Wed Apr 24 12:57:21 UTC 2024 ### Bad Referrer Count: 7104 ### Bad Bot Count: 659 ################################################### @@ -82,6 +82,7 @@ ### 1 = allowed or rate limited less restrictive ### 2 = rate limited more ### 3 = block completely +### 4 = super rate limited ### --------------------------------------------- ### ------------------------------------------------------------ diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 54b864cd5..90fc1b482 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2024.04.4456 -### Updated: Wed Apr 24 12:42:08 UTC 2024 +### Version: V3.2024.04.4457 +### Updated: Wed Apr 24 12:57:21 UTC 2024 ### Bad Referrer Count: 7104 ### Bad Bot Count: 659 ################################################### @@ -82,6 +82,7 @@ ### 1 = allowed or rate limited less restrictive ### 2 = rate limited more ### 3 = block completely +### 4 = super rate limited ### --------------------------------------------- ### ------------------------------------------------------------ diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index c0a378cc6..39719fcf7 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.04.4456 -### Updated: Wed Apr 24 12:42:07 UTC 2024 +### Version: V4.2024.04.4457 +### Updated: Wed Apr 24 12:57:21 UTC 2024 ### Bad Referrer Count: 7104 ### Bad Bot Count: 659 ################################################### @@ -82,6 +82,7 @@ ### 1 = allowed or rate limited less restrictive ### 2 = rate limited more ### 3 = block completely +### 4 = super rate limited ### --------------------------------------------- ### ------------------------------------------------------------ diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index fa7326970..69e464746 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -4456 +4457 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index ad83079b8..90fc1b482 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2024.04.4456 -### Updated: Wed Apr 24 12:42:08 UTC 2024 +### Version: V3.2024.04.4457 +### Updated: Wed Apr 24 12:57:21 UTC 2024 ### Bad Referrer Count: 7104 ### Bad Bot Count: 659 ################################################### diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index bcb409b16..39719fcf7 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.04.4456 -### Updated: Wed Apr 24 12:42:07 UTC 2024 +### Version: V4.2024.04.4457 +### Updated: Wed Apr 24 12:57:21 UTC 2024 ### Bad Referrer Count: 7104 ### Bad Bot Count: 659 ################################################### diff --git a/dev-tools/test1_conf_files/bots.d/blacklist-user-agents.conf b/dev-tools/test1_conf_files/bots.d/blacklist-user-agents.conf index 4e252cb8e..891a389da 100644 --- a/dev-tools/test1_conf_files/bots.d/blacklist-user-agents.conf +++ b/dev-tools/test1_conf_files/bots.d/blacklist-user-agents.conf @@ -5,8 +5,8 @@ # VERSION INFORMATION # #---------------------- -# Version: V4.2022.03 -# Updated: 2022-03-25 +# Version: V4.2024.01 +# Updated: 2024-04-23 #---------------------- # VERSION INFORMATION # @@ -53,12 +53,27 @@ # "~*(?:\b)someverygooduseragentname2(?:\b)" 0; # "~*(?:\b)some\-very\-good\-useragentname2(?:\b)" 0; +# ---------------------- +# RATE LIMITING EXAMPLES +# ---------------------- +# "~*(?:\b)someverybaduseragentname1(?:\b)" 2; +# "~*(?:\b)someverybaduseragentname2(?:\b)" 2; +# "~*(?:\b)some\-very\-bad\-useragentname3(?:\b)" 2; + # --------------------- # BLACKLISTING EXAMPLES # --------------------- -# "~*(?:\b)someverybaduseragentname1(?:\b)" 3; -# "~*(?:\b)someverybaduseragentname2(?:\b)" 3; -# "~*(?:\b)some\-very\-bad\-useragentname2(?:\b)" 3; +# "~*(?:\b)someverybaduseragentname4(?:\b)" 3; +# "~*(?:\b)someverybaduseragentname5(?:\b)" 3; +# "~*(?:\b)some\-very\-bad\-useragentname6(?:\b)" 3; + +# ---------------------------- +# SUPER RATE LIMITING EXAMPLES +# ---------------------------- +# "~*(?:\b)someverybaduseragentname7(?:\b)" 4; +# "~*(?:\b)someverybaduseragentname8(?:\b)" 4; +# "~*(?:\b)some\-very\-bad\-useragentname9(?:\b)" 4; + # Here are some default things I block on my own server, these appear in various types of injection attacks # You can disable them if you have problems or don't agree by switching thir value to 0 or moving them into the whitelist section first and then making their value 0 diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 54c70b229..b419c7ed2 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -172.183.108.38 0; +172.183.162.198 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index c0a378cc6..39719fcf7 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.04.4456 -### Updated: Wed Apr 24 12:42:07 UTC 2024 +### Version: V4.2024.04.4457 +### Updated: Wed Apr 24 12:57:21 UTC 2024 ### Bad Referrer Count: 7104 ### Bad Bot Count: 659 ################################################### @@ -82,6 +82,7 @@ ### 1 = allowed or rate limited less restrictive ### 2 = rate limited more ### 3 = block completely +### 4 = super rate limited ### --------------------------------------------- ### ------------------------------------------------------------ diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 4b0417f44..c20dbc2f7 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.231 1; +10.1.0.33 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 1a7303ece..51c457f9a 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.231 0; +10.1.0.33 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index c0a378cc6..39719fcf7 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.04.4456 -### Updated: Wed Apr 24 12:42:07 UTC 2024 +### Version: V4.2024.04.4457 +### Updated: Wed Apr 24 12:57:21 UTC 2024 ### Bad Referrer Count: 7104 ### Bad Bot Count: 659 ################################################### @@ -82,6 +82,7 @@ ### 1 = allowed or rate limited less restrictive ### 2 = rate limited more ### 3 = block completely +### 4 = super rate limited ### --------------------------------------------- ### ------------------------------------------------------------ diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 4b0417f44..c20dbc2f7 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.231 1; +10.1.0.33 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 1a7303ece..51c457f9a 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.231 0; +10.1.0.33 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index c0a378cc6..39719fcf7 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.04.4456 -### Updated: Wed Apr 24 12:42:07 UTC 2024 +### Version: V4.2024.04.4457 +### Updated: Wed Apr 24 12:57:21 UTC 2024 ### Bad Referrer Count: 7104 ### Bad Bot Count: 659 ################################################### @@ -82,6 +82,7 @@ ### 1 = allowed or rate limited less restrictive ### 2 = rate limited more ### 3 = block completely +### 4 = super rate limited ### --------------------------------------------- ### ------------------------------------------------------------ diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 4b0417f44..c20dbc2f7 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.231 1; +10.1.0.33 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 1a7303ece..51c457f9a 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.231 0; +10.1.0.33 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index c0a378cc6..39719fcf7 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.04.4456 -### Updated: Wed Apr 24 12:42:07 UTC 2024 +### Version: V4.2024.04.4457 +### Updated: Wed Apr 24 12:57:21 UTC 2024 ### Bad Referrer Count: 7104 ### Bad Bot Count: 659 ################################################### @@ -82,6 +82,7 @@ ### 1 = allowed or rate limited less restrictive ### 2 = rate limited more ### 3 = block completely +### 4 = super rate limited ### --------------------------------------------- ### ------------------------------------------------------------ diff --git a/dev-tools/test2_conf_files/bots.d/blacklist-user-agents.conf b/dev-tools/test2_conf_files/bots.d/blacklist-user-agents.conf index 4e252cb8e..891a389da 100644 --- a/dev-tools/test2_conf_files/bots.d/blacklist-user-agents.conf +++ b/dev-tools/test2_conf_files/bots.d/blacklist-user-agents.conf @@ -5,8 +5,8 @@ # VERSION INFORMATION # #---------------------- -# Version: V4.2022.03 -# Updated: 2022-03-25 +# Version: V4.2024.01 +# Updated: 2024-04-23 #---------------------- # VERSION INFORMATION # @@ -53,12 +53,27 @@ # "~*(?:\b)someverygooduseragentname2(?:\b)" 0; # "~*(?:\b)some\-very\-good\-useragentname2(?:\b)" 0; +# ---------------------- +# RATE LIMITING EXAMPLES +# ---------------------- +# "~*(?:\b)someverybaduseragentname1(?:\b)" 2; +# "~*(?:\b)someverybaduseragentname2(?:\b)" 2; +# "~*(?:\b)some\-very\-bad\-useragentname3(?:\b)" 2; + # --------------------- # BLACKLISTING EXAMPLES # --------------------- -# "~*(?:\b)someverybaduseragentname1(?:\b)" 3; -# "~*(?:\b)someverybaduseragentname2(?:\b)" 3; -# "~*(?:\b)some\-very\-bad\-useragentname2(?:\b)" 3; +# "~*(?:\b)someverybaduseragentname4(?:\b)" 3; +# "~*(?:\b)someverybaduseragentname5(?:\b)" 3; +# "~*(?:\b)some\-very\-bad\-useragentname6(?:\b)" 3; + +# ---------------------------- +# SUPER RATE LIMITING EXAMPLES +# ---------------------------- +# "~*(?:\b)someverybaduseragentname7(?:\b)" 4; +# "~*(?:\b)someverybaduseragentname8(?:\b)" 4; +# "~*(?:\b)some\-very\-bad\-useragentname9(?:\b)" 4; + # Here are some default things I block on my own server, these appear in various types of injection attacks # You can disable them if you have problems or don't agree by switching thir value to 0 or moving them into the whitelist section first and then making their value 0 diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 54c70b229..b419c7ed2 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -172.183.108.38 0; +172.183.162.198 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 787d4ebf9..0bd3c0df6 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.04.4456 -### Updated: Wed Apr 24 12:42:07 UTC 2024 +### Version: V4.2024.04.4457 +### Updated: Wed Apr 24 12:57:21 UTC 2024 ### Bad Referrer Count: 7104 ### Bad Bot Count: 659 ################################################### @@ -82,6 +82,7 @@ ### 1 = allowed or rate limited less restrictive ### 2 = rate limited more ### 3 = block completely +### 4 = super rate limited ### --------------------------------------------- ### ------------------------------------------------------------ diff --git a/dev-tools/test3_conf_files/bots.d/blacklist-user-agents.conf b/dev-tools/test3_conf_files/bots.d/blacklist-user-agents.conf index 4e252cb8e..891a389da 100644 --- a/dev-tools/test3_conf_files/bots.d/blacklist-user-agents.conf +++ b/dev-tools/test3_conf_files/bots.d/blacklist-user-agents.conf @@ -5,8 +5,8 @@ # VERSION INFORMATION # #---------------------- -# Version: V4.2022.03 -# Updated: 2022-03-25 +# Version: V4.2024.01 +# Updated: 2024-04-23 #---------------------- # VERSION INFORMATION # @@ -53,12 +53,27 @@ # "~*(?:\b)someverygooduseragentname2(?:\b)" 0; # "~*(?:\b)some\-very\-good\-useragentname2(?:\b)" 0; +# ---------------------- +# RATE LIMITING EXAMPLES +# ---------------------- +# "~*(?:\b)someverybaduseragentname1(?:\b)" 2; +# "~*(?:\b)someverybaduseragentname2(?:\b)" 2; +# "~*(?:\b)some\-very\-bad\-useragentname3(?:\b)" 2; + # --------------------- # BLACKLISTING EXAMPLES # --------------------- -# "~*(?:\b)someverybaduseragentname1(?:\b)" 3; -# "~*(?:\b)someverybaduseragentname2(?:\b)" 3; -# "~*(?:\b)some\-very\-bad\-useragentname2(?:\b)" 3; +# "~*(?:\b)someverybaduseragentname4(?:\b)" 3; +# "~*(?:\b)someverybaduseragentname5(?:\b)" 3; +# "~*(?:\b)some\-very\-bad\-useragentname6(?:\b)" 3; + +# ---------------------------- +# SUPER RATE LIMITING EXAMPLES +# ---------------------------- +# "~*(?:\b)someverybaduseragentname7(?:\b)" 4; +# "~*(?:\b)someverybaduseragentname8(?:\b)" 4; +# "~*(?:\b)some\-very\-bad\-useragentname9(?:\b)" 4; + # Here are some default things I block on my own server, these appear in various types of injection attacks # You can disable them if you have problems or don't agree by switching thir value to 0 or moving them into the whitelist section first and then making their value 0 diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 54c70b229..b419c7ed2 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -172.183.108.38 0; +172.183.162.198 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index eb07a837b..04d28f0a2 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.04.4456 -### Updated: Wed Apr 24 12:42:07 UTC 2024 +### Version: V4.2024.04.4457 +### Updated: Wed Apr 24 12:57:21 UTC 2024 ### Bad Referrer Count: 7104 ### Bad Bot Count: 659 ################################################### @@ -82,6 +82,7 @@ ### 1 = allowed or rate limited less restrictive ### 2 = rate limited more ### 3 = block completely +### 4 = super rate limited ### --------------------------------------------- ### ------------------------------------------------------------ diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index 4b0417f44..c20dbc2f7 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.231 1; +10.1.0.33 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 912d7a45d..3b5db21f4 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Aboundex -DigitalPebble +BBBike +Blackboard +Citoid +IDBot +Majestic12 MauiBot -MolokaiBot -NetLyzer -RedesScrapy -SemanticJuice -SiteLockSpider -Snapbot -crawler4j +SalesIntelligent +SemrushBot +Sogou web spider +Zade diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 3e8182d47..f80bc8760 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,100 +1,100 @@ +404checker +ALittle Client Acunetix -Alligator -AlphaBot -Apexoo +AdsTxtCrawlerTP +Aipbot +Alexibot +Anarchy99 +Aspiegel Attach -BacklinkCrawler -Badass +AwarioBot +AwarioSmartBot +Bandit Battleztar Bazinga Black Hole Blow -BunnySlippers -BuzzSumo -Bytespider -CensysInspect -Crawling at Home Project -DBLBot -DomainSigmaCrawler -ECCP/1.0 -Ecxi -ExtractorPro +Boardreader +CODE87 +ChatGPT-User +CheeseBot +ChinaClaw +Cogentbot +CyotekWebCopy +Discobot +Dragonfly +EroCrawler +ExtLinksBot +Extractor Extreme Picture Finder Ezooms -Fimap -FrontPage -GrapeFX -GrapeshotCrawler -GridBot -Haansoft -HaosouSpider +FHscan +Firefox/7.0 +Go-Ahead-Got-It +Harvest +Havij +HonoluluBot IRLbot -Image Fetch -InfoNaviRobot -IstellaBot -JennyBot -Joomla -Keyword Density -LinkWalker -MJ12bot -Mag-Net -MarkWatch -Mojeek +IndeedBot +Internet Ninja +LeechFTP +LeechGet +LexiBot +Libwhisker +LinkScan +LinqiaScrapeBot +Majestic-SEO +Masscan +MauiBot +MeanPath Bot +MegaIndex.ru +MicroMessenger +Microsoft URL Control +Minefield +Moblie Safari Morfeus Fucking Scanner -Navroad -NetAnts -NetMechanic -NetSpider -OpenVAS +Nibbler OrangeBot -PageAnalyzer -Pockey -PyCurl -RedesScrapy -RepoMonkey -Ripper -SEOkicks-Robot -Scanbot -Scrapy -ScreenerBot +Panscient +ProPowerBot +Probethenet +QueryN Metasearch +Rainbot +ReGet +Searchestate SearchmetricsBot -Shodan -Sosospider -Sottopop -Sqlworm -T8Abot +SemanticJuice +SeobilityBot +Site Sucker +SiteSucker +Snapbot +Spammen +T0PHackTeam Teleport -The Intraformant -Toweyabot -Turingos -URLy Warning -URLy.Warning +Telesphorep +TheNomad +TurnitinBot +TwengaBot UnisterBot -VoidEYE -WWW-Mechanize +Upflow +WeSEE +Web Enhancer Web Fetch -Web Pix -WebBandit -WebFetch -WebFuck -WebGo IS -WebLeacher -WebReaper +Web Fuck WebWhacker -WebZIP Webalta -Webshag +WebsiteQuester +Whack +WiseGuys Robot +Wotbox Xaldon WebSpider -Xenu -YaK +Xaldon_WebSpider +Zade ZyBorg archive.org_bot -demandbase-bot -eCatch -facebookscraper -iubenda-radar +isitwp.com +linkdexbot lwp-trivial -ripz -sp_auditbot -trendiction.com -webgains-bot -zauba.io +netEstate NE Crawler +probe-image-size +seostar +zgrab diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 1fb9780a7..8348f0338 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -bestprofits-there.com -fanrto.com -gamewrath.com -macotool.com -mixx.com -prostitutki-tolyatti.org -seminarygeorgia59.ga -shoesonlinebuy.cn -thin.me.pn -tizanidinezanaflex.blogspot.com +adamoads.com +cialiswithoutadoctor.net +dorratex.tn +freetangodownload.com +igru-xbox.net +nuker.com +steame.ru +supervesti.ru +theprofitsmaker.net +zona-aqua.ru diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index ed30f9a7b..c4f95f20b 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,100 +1,100 @@ -883zy.com -adservme.com -adverster.com -affrh2015.com -agadelha.com.br -aihelen.net -aksonural.ru -anatomiy.com -apple.com-cleaner.systems -arkkivoltti.net -babieca.com -blockworld.ru -buqyxa.rincian.info -buytizanidineonlinenoprescription.blogspot.com -cam-kontakte.org -carson.getenjoyment.net -cartierbracelet.xyz -cartierloveringreplica.xyz -cigarpass.com -clothesforcash.com -coaching-netz.info -coderstate.com -com-supportcenter.website -costablue.xyz -creditmoney.com.ua -d2jsp.org -dev-seo.blog -diffbot.com -domik-derevne.ru -dprtb.com -erotik0049.com -euroskat.ru +12bet.com +51.la +absolute-s.ru +acads.net +aero2.ru +aerogo.com +alfa9.com +alkoravto.ru +allkrim.com +amehdaily.com +anaksma.info +arclk.net +azartniy-bonus.com +blogrankers.com +buttons-for-website.com +buynorxx.com +buypillsorderonline.com +canvas.gloverid.site +cardsdumps.com +ciproandtizanidine.blogspot.com +clayvasedesigns.tk +codysbbq.com +contentlook.co +cool-wedding.net +deshevo-nedorogo.ru +designdevise.com +dieswaene.com +dostavimvdom.ru +doxyporno.com +electronicadirect.com +ereko.ru +ezigaretteshop.webs.com +farmprofi.net fillmewithhappiness.com -findpics.pw -free-share-buttons.blogspot.com -funtoonez.com -gabeshop.ru -gepezz.info -harmonyglen.us -homedo.fabpage.com -infowarcraft.ru -iqs.biz.ua -isistaylorporn.info -istock-mebel.ru -istripper.com -jewelryandfiligree.com -justprofit.xyz -larose.jb2c.me -lego4x4.ru -limtu.ifmo.ru -mcnamaratech.com -mowser.com -musflashtv.com -my-aladin.com -myphotopipe.com -nudo.ca -okonich.com.ua -online-hd.pl -osoznanie-narkotikam.net -pl.aasoldes.fr -plastweb.ru -polybuild.ru -prizesbook.online -pro-okis.ru -producm.ru -prostitutki-novgoroda.org -qitt.ru -rcpmda.ikan1080.xyz -redirectme.net -reining.lovasszovetseg.hu -robotixix.com -runtnc.net -rutor.vip -s.lollypopgaming.com -saratov.xrus.org -seobility.net -seoheap.com -sex-znakomstva.online -sirpornogratis.xxx -sitevaluation.com -stairliftstrue.com -starpages.net +filunika.com.ru +formulaantiuban.com +fortwosmartcar.pw +genericviagrasildenafiled.net +ghernnqr.skyrock.com +glasof.es +goroda-vsego-mira.ru +herehloadibs.cf +hotsocialz.com +itsdp3.com +kino2018.cc +konica.kz +letsrepair.in +lirunet.ru +littlesexdolls.com +lowephotos.info +macnewtech.com +maridan.com.ua +mobilierland.com +modenamebel.ru +mojpregled.com +mriyadh.com +myfreemp3.eu +myonlinepayday.co +myprintscreen.com +newstudio.tv +ningessaybe.me +no-rx.info +nosecret.com.ua +nzfilecloud.weebly.com +ohmyrings.com +ok.ru +origin-my.ru +paphoselectricianandplumber.com +pl-vouchers.com +poquoson.org +privatefx.all4invest.info +proftests.net +prokotov.com +sad-torg.com.ua +sexvporno.ru +sexyteens.hol.es +small-game.com +socialmadesimple.com +speeddream.xyz +superoboi.com.ua +syvertsen-da.no +t-bygg.com +tasteidea.com thedownloadfromwarez.blogspot.com -tizanidinehcl2mgtablet.blogspot.com -topsiteminecraft.com -track.deriv.com -trkdf.com -tvoystartup.ru -ufa.xrus.org -vk-mus.ru -vonradio.com -webs.com -webshoppermac.com -wholesalecheapjerseysfree.com -wonderfulflowers.biz -xchangetrak.com -za-fun-offer.com -zakazfutbolki.com -zlatnajesen.com -znakomstvaonlain.ru +tizanidineoral.blogspot.com +topclickguru.com +tubeline.biz +umaseh.com +uptimechecker.com +usadacha.net +uvozdeckych.info +videochat.mx +whatsappbot.flyland.ru +xn--80aaajbdbddwj2alwjieei2afr3v.xn--p1ai +xxart.ru +xzlive.com +yoopsie.com +you-shall-not-pass.is74.ru +yourothersite.com +znakomstva-moskva77.ru diff --git a/dev-tools/test_units/ratelimittest-master.txt b/dev-tools/test_units/ratelimittest-master.txt index 45d3cbc77..a4520e06f 100644 --- a/dev-tools/test_units/ratelimittest-master.txt +++ b/dev-tools/test_units/ratelimittest-master.txt @@ -1,14 +1,4 @@ - - - -Welcome to NginX - - -
-

Welcome to the NginX Ultimate Bad Bot Blocker

-

This is merely used for testing !!!

- -ml> + 503 Service Temporarily Unavailable

503 Service Temporarily Unavailable

@@ -651,4 +641,14 @@

Welcome to the NginX Ultimate Bad Bot Blocker

This is merely used for testing !!!

+ + + +Welcome to NginX + + +
+

Welcome to the NginX Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ \ No newline at end of file diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index 1a7303ece..51c457f9a 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.231 0; +10.1.0.33 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index d584d6fc2..a000aff4a 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -6,8 +6,8 @@ ### Version Information # ################################################### -### Version: V4.2024.04.4456 -### Updated: Wed Apr 24 12:42:09 UTC 2024 +### Version: V4.2024.04.4457 +### Updated: Wed Apr 24 12:57:22 UTC 2024 ### Bad Bot Count: 659 ################################################### ### Version Information ## diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index d584d6fc2..a000aff4a 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,8 +6,8 @@ ### Version Information # ################################################### -### Version: V4.2024.04.4456 -### Updated: Wed Apr 24 12:42:09 UTC 2024 +### Version: V4.2024.04.4457 +### Updated: Wed Apr 24 12:57:22 UTC 2024 ### Bad Bot Count: 659 ################################################### ### Version Information ##