From 84bb5fe5ef72b4b20eb47072779881ccd7ce2ec9 Mon Sep 17 00:00:00 2001 From: mitchellkrogza Date: Sun, 28 Jul 2024 06:56:33 +0000 Subject: [PATCH] V4.2024.07.4651 [ci skip] --- AUTO-CONFIGURATION.md | 4 +- MANUAL-CONFIGURATION.md | 4 +- README.md | 4 +- _generator_lists/bad-user-agents.list | 4 +- _google_analytics_ghost_spam/README.md | 4 +- .../etc/nginx/conf.d/globalblacklist.conf | 8 +- conf.d/globalblacklist-testing-version.conf | 8 +- conf.d/globalblacklist.conf | 8 +- dev-tools/_robots_input/robots-input.txt | 2 + dev-tools/buildnumber | 2 +- dev-tools/globalblacklist-testing.template | 8 +- dev-tools/globalblacklist.template | 8 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 8 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 8 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 8 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/blacklist-user-agents.conf | 2 + .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 8 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 8 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 8 +- dev-tools/test_units/blacklist-ips.conf | 2 +- .../test_units/blacklist-user-agents.conf | 2 + .../random-bots-for-test-quick.list | 20 +- .../random-bots-for-whitelist-test.list | 172 +++++++-------- .../random-referrers-for-test-quick.list | 20 +- .../random-referrers-for-whitelist-test.list | 198 +++++++++--------- dev-tools/test_units/ratelimittest-master.txt | 97 +++------ dev-tools/test_units/whitelist-ips.conf | 2 +- dev-tools/tmp/robots.txt | 10 +- robots.txt/robots.txt | 10 +- 38 files changed, 333 insertions(+), 332 deletions(-) diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index b70a9e337..5965e1ade 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2024.07.4650 +#### Version: V4.2024.07.4651 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [665](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [667](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index b2e0866b5..3bdc6a14a 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2024.07.4650 +#### Version: V4.2024.07.4651 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [665](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [667](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index 3f4d8ba66..646177695 100755 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2024.07.4650 +#### Version: V4.2024.07.4651 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [665](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [667](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list index 87b393d1c..5cb322a2c 100755 --- a/_generator_lists/bad-user-agents.list +++ b/_generator_lists/bad-user-agents.list @@ -1,5 +1,3 @@ -Spider_Bot/3.0 -Spider_Bot 01h4x.com 360Spider 404checker @@ -455,6 +453,8 @@ Spammen SpankBot Spanner Spbot +Spider_Bot +Spider_Bot/3.0 Spinn3r SputnikBot Sqlmap diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 78d172d38..f6a728e04 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2024.07.4650 +#### Version: V4.2024.07.4651 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [665](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [667](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index fec1aca59..48f3f9aab 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.07.4650 -### Updated: Sat Jul 27 22:01:11 UTC 2024 +### Version: V4.2024.07.4651 +### Updated: Sun Jul 28 06:52:49 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 665 +### Bad Bot Count: 667 ################################################### ### VERSION INFORMATION ## @@ -606,6 +606,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)SpankBot(?:\b)" 3; "~*(?:\b)Spanner(?:\b)" 3; "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spider_Bot(?:\b)" 3; + "~*(?:\b)Spider_Bot/3.0(?:\b)" 3; "~*(?:\b)Spinn3r(?:\b)" 3; "~*(?:\b)SputnikBot(?:\b)" 3; "~*(?:\b)Sqlmap(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index ffe975ba3..838994244 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2024.07.4650 -### Updated: Sat Jul 27 22:01:12 UTC 2024 +### Version: V3.2024.07.4651 +### Updated: Sun Jul 28 06:52:50 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 665 +### Bad Bot Count: 667 ################################################### ### VERSION INFORMATION ## @@ -603,6 +603,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)SpankBot(?:\b)" 3; "~*(?:\b)Spanner(?:\b)" 3; "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spider_Bot(?:\b)" 3; + "~*(?:\b)Spider_Bot/3.0(?:\b)" 3; "~*(?:\b)Spinn3r(?:\b)" 3; "~*(?:\b)SputnikBot(?:\b)" 3; "~*(?:\b)Sqlmap(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index fec1aca59..48f3f9aab 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.07.4650 -### Updated: Sat Jul 27 22:01:11 UTC 2024 +### Version: V4.2024.07.4651 +### Updated: Sun Jul 28 06:52:49 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 665 +### Bad Bot Count: 667 ################################################### ### VERSION INFORMATION ## @@ -606,6 +606,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)SpankBot(?:\b)" 3; "~*(?:\b)Spanner(?:\b)" 3; "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spider_Bot(?:\b)" 3; + "~*(?:\b)Spider_Bot/3.0(?:\b)" 3; "~*(?:\b)Spinn3r(?:\b)" 3; "~*(?:\b)SputnikBot(?:\b)" 3; "~*(?:\b)Sqlmap(?:\b)" 3; diff --git a/dev-tools/_robots_input/robots-input.txt b/dev-tools/_robots_input/robots-input.txt index 70aca53c1..7ae4bf3ce 100644 --- a/dev-tools/_robots_input/robots-input.txt +++ b/dev-tools/_robots_input/robots-input.txt @@ -453,6 +453,8 @@ Spammen SpankBot Spanner Spbot +Spider_Bot +Spider_Bot/3.0 Spinn3r SputnikBot Sqlmap diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index c35025331..21f27904a 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -4650 +4651 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index ffe975ba3..838994244 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2024.07.4650 -### Updated: Sat Jul 27 22:01:12 UTC 2024 +### Version: V3.2024.07.4651 +### Updated: Sun Jul 28 06:52:50 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 665 +### Bad Bot Count: 667 ################################################### ### VERSION INFORMATION ## @@ -603,6 +603,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)SpankBot(?:\b)" 3; "~*(?:\b)Spanner(?:\b)" 3; "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spider_Bot(?:\b)" 3; + "~*(?:\b)Spider_Bot/3.0(?:\b)" 3; "~*(?:\b)Spinn3r(?:\b)" 3; "~*(?:\b)SputnikBot(?:\b)" 3; "~*(?:\b)Sqlmap(?:\b)" 3; diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index fec1aca59..48f3f9aab 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.07.4650 -### Updated: Sat Jul 27 22:01:11 UTC 2024 +### Version: V4.2024.07.4651 +### Updated: Sun Jul 28 06:52:49 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 665 +### Bad Bot Count: 667 ################################################### ### VERSION INFORMATION ## @@ -606,6 +606,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)SpankBot(?:\b)" 3; "~*(?:\b)Spanner(?:\b)" 3; "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spider_Bot(?:\b)" 3; + "~*(?:\b)Spider_Bot/3.0(?:\b)" 3; "~*(?:\b)Spinn3r(?:\b)" 3; "~*(?:\b)SputnikBot(?:\b)" 3; "~*(?:\b)Sqlmap(?:\b)" 3; diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 9a5e0240d..a97232f96 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -74.249.14.222 0; +20.81.158.161 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index fec1aca59..48f3f9aab 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.07.4650 -### Updated: Sat Jul 27 22:01:11 UTC 2024 +### Version: V4.2024.07.4651 +### Updated: Sun Jul 28 06:52:49 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 665 +### Bad Bot Count: 667 ################################################### ### VERSION INFORMATION ## @@ -606,6 +606,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)SpankBot(?:\b)" 3; "~*(?:\b)Spanner(?:\b)" 3; "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spider_Bot(?:\b)" 3; + "~*(?:\b)Spider_Bot/3.0(?:\b)" 3; "~*(?:\b)Spinn3r(?:\b)" 3; "~*(?:\b)SputnikBot(?:\b)" 3; "~*(?:\b)Sqlmap(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 8876c32f2..d6d997e9e 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.8 1; +10.1.0.95 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index a7e4dac00..5ebd2f7e4 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.8 0; +10.1.0.95 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index fec1aca59..48f3f9aab 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.07.4650 -### Updated: Sat Jul 27 22:01:11 UTC 2024 +### Version: V4.2024.07.4651 +### Updated: Sun Jul 28 06:52:49 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 665 +### Bad Bot Count: 667 ################################################### ### VERSION INFORMATION ## @@ -606,6 +606,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)SpankBot(?:\b)" 3; "~*(?:\b)Spanner(?:\b)" 3; "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spider_Bot(?:\b)" 3; + "~*(?:\b)Spider_Bot/3.0(?:\b)" 3; "~*(?:\b)Spinn3r(?:\b)" 3; "~*(?:\b)SputnikBot(?:\b)" 3; "~*(?:\b)Sqlmap(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 8876c32f2..d6d997e9e 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.8 1; +10.1.0.95 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index a7e4dac00..5ebd2f7e4 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.8 0; +10.1.0.95 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index fec1aca59..48f3f9aab 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.07.4650 -### Updated: Sat Jul 27 22:01:11 UTC 2024 +### Version: V4.2024.07.4651 +### Updated: Sun Jul 28 06:52:49 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 665 +### Bad Bot Count: 667 ################################################### ### VERSION INFORMATION ## @@ -606,6 +606,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)SpankBot(?:\b)" 3; "~*(?:\b)Spanner(?:\b)" 3; "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spider_Bot(?:\b)" 3; + "~*(?:\b)Spider_Bot/3.0(?:\b)" 3; "~*(?:\b)Spinn3r(?:\b)" 3; "~*(?:\b)SputnikBot(?:\b)" 3; "~*(?:\b)Sqlmap(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 8876c32f2..d6d997e9e 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.8 1; +10.1.0.95 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index de21db88e..850701628 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -514,6 +514,8 @@ "~*(?:\b)SpankBot(?:\b|)" 0; "~*(?:\b)Spanner(?:\b|)" 0; "~*(?:\b)Spbot(?:\b|)" 0; + "~*(?:\b)Spider_Bot(?:\b|)" 0; + "~*(?:\b)Spider_Bot/3.0(?:\b|)" 0; "~*(?:\b)Spinn3r(?:\b|)" 0; "~*(?:\b)SputnikBot(?:\b|)" 0; "~*(?:\b)Sqlmap(?:\b|)" 0; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index a7e4dac00..5ebd2f7e4 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.8 0; +10.1.0.95 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index fec1aca59..48f3f9aab 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.07.4650 -### Updated: Sat Jul 27 22:01:11 UTC 2024 +### Version: V4.2024.07.4651 +### Updated: Sun Jul 28 06:52:49 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 665 +### Bad Bot Count: 667 ################################################### ### VERSION INFORMATION ## @@ -606,6 +606,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)SpankBot(?:\b)" 3; "~*(?:\b)Spanner(?:\b)" 3; "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spider_Bot(?:\b)" 3; + "~*(?:\b)Spider_Bot/3.0(?:\b)" 3; "~*(?:\b)Spinn3r(?:\b)" 3; "~*(?:\b)SputnikBot(?:\b)" 3; "~*(?:\b)Sqlmap(?:\b)" 3; diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 9a5e0240d..a97232f96 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -74.249.14.222 0; +20.81.158.161 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 8072cae2c..70a5e1c48 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.07.4650 -### Updated: Sat Jul 27 22:01:11 UTC 2024 +### Version: V4.2024.07.4651 +### Updated: Sun Jul 28 06:52:49 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 665 +### Bad Bot Count: 667 ################################################### ### VERSION INFORMATION ## @@ -606,6 +606,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)SpankBot(?:\b)" 3; "~*(?:\b)Spanner(?:\b)" 3; "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spider_Bot(?:\b)" 3; + "~*(?:\b)Spider_Bot/3.0(?:\b)" 3; "~*(?:\b)Spinn3r(?:\b)" 3; "~*(?:\b)SputnikBot(?:\b)" 3; "~*(?:\b)Sqlmap(?:\b)" 3; diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 9a5e0240d..a97232f96 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -74.249.14.222 0; +20.81.158.161 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 109b62469..b0301db1f 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2024.07.4650 -### Updated: Sat Jul 27 22:01:11 UTC 2024 +### Version: V4.2024.07.4651 +### Updated: Sun Jul 28 06:52:49 UTC 2024 ### Bad Referrer Count: 7104 -### Bad Bot Count: 665 +### Bad Bot Count: 667 ################################################### ### VERSION INFORMATION ## @@ -606,6 +606,8 @@ map $http_user_agent $bad_bot { "~*(?:\b)SpankBot(?:\b)" 3; "~*(?:\b)Spanner(?:\b)" 3; "~*(?:\b)Spbot(?:\b)" 3; + "~*(?:\b)Spider_Bot(?:\b)" 3; + "~*(?:\b)Spider_Bot/3.0(?:\b)" 3; "~*(?:\b)Spinn3r(?:\b)" 3; "~*(?:\b)SputnikBot(?:\b)" 3; "~*(?:\b)Sqlmap(?:\b)" 3; diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index 8876c32f2..d6d997e9e 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.8 1; +10.1.0.95 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/blacklist-user-agents.conf b/dev-tools/test_units/blacklist-user-agents.conf index de21db88e..850701628 100644 --- a/dev-tools/test_units/blacklist-user-agents.conf +++ b/dev-tools/test_units/blacklist-user-agents.conf @@ -514,6 +514,8 @@ "~*(?:\b)SpankBot(?:\b|)" 0; "~*(?:\b)Spanner(?:\b|)" 0; "~*(?:\b)Spbot(?:\b|)" 0; + "~*(?:\b)Spider_Bot(?:\b|)" 0; + "~*(?:\b)Spider_Bot/3.0(?:\b|)" 0; "~*(?:\b)Spinn3r(?:\b|)" 0; "~*(?:\b)SputnikBot(?:\b|)" 0; "~*(?:\b)Sqlmap(?:\b|)" 0; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index e3c2303b2..84e5d3f6f 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Barkrowler -Boardreader -BunnySlippers -Calculon -CheeseBot -DnBCrawler-Analytics -Haansoft -Mass Downloader -Toweyabot -webmeup-crawler +Bandit +Blackboard +Cocolyzebot +Copier +Download Wonder +Freeuploader +HMView +Kinza +Offline Explorer +Typhoeus diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 50715f7bc..2f1834ef1 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,100 +1,100 @@ -AIBOT -ALittle Client -AhrefsBot -Anarchy99 -BLEXBot -BackStreet -Bitacle -BlowFish -Buddy -CATExplorador -CrunchBot +360Spider +Atomseobot +Attach +BDFetch +BackDoorBot +Badass +Barkrowler +Black Hole +Boardreader +Brandwatch +Buck +BuiltWith +CazoodleBot +CheTeam +CheeseBot +ChinaClaw +Crescent +Curious +Disco DomCopBot -Drip -ECCP/1.0 -EMail Siphon -Evil -Fyrebot -G-i-g-a-b-o-t -Go!Zilla -GrapeFX -Haansoft +DomainSigmaCrawler +Domains Project +Dotbot +Extractor +ExtractorPro +FHscan +Firefox/7.0 +GermCrawler +Grabber +Harvest Heritrix -Hloader -HonoluluBot IDBot -Image Fetch -Indy Library -InternetMeasurement -Jyxobot -Kinza -Lanshanbot -Leap -LeechGet -Linkbot -LinqiaRSSBot -MFC_Tear_Sample -Magnet -Majestic SEO -Masscan -Mr.4x3 -NPbot -Name Intelligence +Image Sucker +Internet Ninja +JetCar +JustView +Kozmosbot +LeechFTP +Lftp +Libwhisker +Lightspeedsystems +Majestic12 +MarkMonitor +NICErsPRO +Nameprotect NearSite -Netcraft -Netvibes -Niki-bot -Nikto -Nuclei -Nutch -Openfind -OutclicksBot -OutfoxBot -POE-Component-Client-HTTP -Papa Foto -Picscout -ProWebWalker -RSSingBot +Needle +NetSpider +Nimbostratus +PECL::HTTP +PeoplePal +Pi-Monster +Piepmatz +ProPowerBot +Pump +Rainbot +RankActive Rankivabot -RebelMouse -SearchmetricsBot -SemanticJuice -SeobilityBot -SiteLockSpider -SiteSnagger -SlySearch +RocketCrawler +SEOlyticsCrawler +ScreenerBot +ScrepyBot +SeekportBot +Siphon Snapbot -Snoopy +Sociscraper Spammen -SuperBot -SurveyBot -Suzuran -TightTwatBot -Toata -Turnitin -URLy.Warning -Upflow +Spanner +SputnikBot +Teleport +Thumbor +TinyTestBot +Twice +VCI Vacuum -VelenPublicWebCrawler VeriCiteCrawler -WASALive-Bot +Voil WWW-Collector-E -WWW-Mechanize -WWW::Mechanize -Web Pix -WebFuck -WebGo IS +Web Sucker +WebCopier WebLeacher -WebWhacker -Website Quester -Whatweb -Zeus -adscanner +WebPix +WebSauger +WebsiteQuester +Xenu +Zade +ZoominfoBot +arquivo-web-crawler +awario.com cah.io.community -evc-batch +crawl.sogou.com +gopher +ips-agent lwp-request -s1z.ru -tAkeOut -webgains-bot -webpros.com +probe-image-size +seocompany.store +trendiction.de +webmeup-crawler webprosbot +zauba.io diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 67e48547d..959df209a 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -helpmymacfaster.trade -importchinacoach-teach.com -insurple.com -most-kerch.org -picscout.com -start.myplaycity.com -wstroika.ru -your-bearings.com -zoominfo.com -zscaler.net +elmacho.xyz +encodable.com +falcoware.com +gifspics.com +itrevolution.cf +kinocccp.net +nextrent-crimea.ru +sendearnings.com +yadro.ru +yginekologa.com diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index a950e947f..e53bf7e40 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,100 +1,100 @@ -01apple.com -4webmasters.com -acads.net -adprotect.net -advertex.info -ageofclones.com -agroeconom.kz -autorn.ru -b2b-lounge.com -babs.com.ua -babyfactory.fr -beachdriveblog.com -beetpics.pw -besthoro.ru -besttorrentknifta.weebly.com -blue-square.biz -buqayy0.livejournal.com -cam-kontakte.org -casinos4dummies.co.uk -cheapsergiorossi.online -cigarpass.com -clickpapa.com -community.allhiphop.com -compliance-nelson.top -cornomase.win -crazyboost.pro -db.speedup.gdn -ddpills.com -dev-seo.blog -divatraffic.com -docsportal.net -dwomlink.info +883zy.com +abcdefh.xyz +admeasures.com +adserver-e7.com +anonymousfox.co +apiholdingmypage-a.akamaihd.net +apioasisspacenet-a.akamaihd.net +arabgirls.adultgalls.com +arcteryxstore.online +azlex.uz +babieca.com +bannerads.de +beachtoday.ru +best-deals-products.com +bezsporno.ru +bnt-team.com +brakehawk.com +breastaugmentation.co.za +buron.pw +buy-cheap-pills-order-online.com +californianews.cf +centraletermice.us +chidporn.com +chimiver.info +coldfilm.ru +decorazilla.com +djonwatch.ru +dogoimage.com +eager-nash.188-93-233-196.plesk.page +ec-file.info +editors.choice6912650.hulfingtonpost.com ednorxmed.com -everypony.ru -evidencecleanergold.com -ezrvrentals.com -foxweber.com -gallerily.com -gambarxkata.co -gay-file.com -getoutofdebtfree.org -ghernnqr.skyrock.com -glogow.pl -googleseo.com.tr -guarrasdelporno.xxx -hacktougroup.ru -hatedriveapart.com -hd-film.pl -hobbyhuren24.net -homedecorpicture.us -indetiske.ya.ru -infobabki.ru -kaz.kz -kidd.reunionwatch.com -kinogolos.ru -kinosed.net -kosynka-games.ru -kustanay.kz -laulini.soclog.se -littleberry.ru -mnogolok.info -napalm51.nut.cc -nuker.com -nzfilecloud.weebly.com -oballergiya.ru -ok-ua.info -okroshki.ru -pay2me.pl -piter.xrus.org -popugauka.ru -pornokorol.com -produkto.net -pronorm.fr -rankingchart.de -rating-casino2021.ru -renhacklids.tk -rentehno.ru -responsive-test.net -runofilms.ru -sabizonline.com -sajatvelemeny.com -seasaltwithfood.com -seoboxes.com -shtora66.ru -stariy-baku.com -syvertsen-da.no -szamponrevita.pl -takeprofitsystem.com -time-japan.ru -tracksz.co -ufa.xrus.org -urldelivery.com -varbergsvind.se -watracker.net -webcamtalk.net -wesharepics.site -works.if.ua -xsfetish.org -youtube-downloader.savetubevideo.com -zambini.ru +ege-essay.ru +enbersoft.com +evening-dating-club.info +freesitetest.com +freewlan.info +girls-ufa.ru +greatdealshop.com +hamptonoaks.ca +hotchatdate.com +isoveti.ru +kellyonline.xyz +kiinomaniak.pl +louboutinshoes.xyz +lunamedia.co +mabdoola.blogspot.com +maranbrinfo.com.br +marmitaco.cat +mcadamssupplyco.com +mielec.pl +mirtorrent.net +music.utrolive.ru +nailsimg.com +nhl09.ru +one.net.in +onlyporno.ru +ooo-gotovie.ru +papasdelivery.ru +picanalyzer.data-ox.com +pihl.se +pinapchik.com +pizza-tycoon.com +popads.net +prodvigator.ua +promgirldresses.xyz +pronekut.com +qwarckoine.com +responsinator.com +sbricur.com +scansafe.net +seminarygeorgia59.ga +sexvporno.ru +shodanhq.com +smsactivator.ru +snip.to +social-button.xyz +stardevine.com +stroydetali.ru +titslove.yopoint.in +tizanidinerestlesslegsyndrome.blogspot.com +tk-assortiment.ru +topbestgames.com +torrent-to-magnet.com +trichizobswiv.agddns.net +turvgori.ru +urlopener.blogspot.com.au +vereo.eu +videochat.cafe +visa-pasport.ru +watchmyfb.pl +webalan.ru +website-speed-check.site +whosonmyserver.com +wma-x.com +x-diesel.com +xn----btbdvdh4aafrfciljm6k.xn--p1ai +zoominfo.com +zoompegs.com +zryydi.com diff --git a/dev-tools/test_units/ratelimittest-master.txt b/dev-tools/test_units/ratelimittest-master.txt index d2f03ce56..0dccbaf08 100644 --- a/dev-tools/test_units/ratelimittest-master.txt +++ b/dev-tools/test_units/ratelimittest-master.txt @@ -8,64 +8,6 @@

Welcome to the NginX Ultimate Bad Bot Blocker

This is merely used for testing !!!

-OCTYPE html> - - -Welcome to NginX - - -
-

Welcome to the NginX Ultimate Bad Bot Blocker

-

This is merely used for testing !!!

- - -503 Service Temporarily Unavailable - -

503 Service Temporarily Unavailable

-
nginx/1.18.0 (Ubuntu)
- - - -503 Service Temporarily Unavailable - -

503 Service Temporarily Unavailable

-
nginx/1.18.0 (Ubuntu)
- - - - - -Welcome to NginX - - -
-

Welcome to the NginX Ultimate Bad Bot Blocker

-

This is merely used for testing !!!

- - -503 Service Temporarily Unavailable - -

503 Service Temporarily Unavailable

-
nginx/1.18.0 (Ubuntu)
- - - -503 Service Temporarily Unavailable - -

503 Service Temporarily Unavailable

-
nginx/1.18.0 (Ubuntu)
- - - - - -Welcome to NginX - - -
-

Welcome to the NginX Ultimate Bad Bot Blocker

-

This is merely used for testing !!!

- @@ -76,14 +18,37 @@

Welcome to the NginX Ultimate Bad Bot Blocker

This is merely used for testing !!!

- -503 Service Temporarily Unavailable - -

503 Service Temporarily Unavailable

-
nginx/1.18.0 (Ubuntu)
- - - + + + +Welcome to NginX + + +
+

Welcome to the NginX Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + + + +Welcome to NginX + + +
+

Welcome to the NginX Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + + + +Welcome to NginX + + +
+

Welcome to the NginX Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + Welcome to NginX diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index a7e4dac00..5ebd2f7e4 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.8 0; +10.1.0.95 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index a3458e1d5..31500b020 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2024.07.4650 -### Updated: Sat Jul 27 22:01:12 UTC 2024 -### Bad Bot Count: 665 +### Version: V4.2024.07.4651 +### Updated: Sun Jul 28 06:52:50 UTC 2024 +### Bad Bot Count: 667 ################################################### ### Version Information ## @@ -925,6 +925,10 @@ User-agent: Spanner Disallow:/ User-agent: Spbot Disallow:/ +User-agent: Spider_Bot +Disallow:/ +User-agent: Spider_Bot/3.0 +Disallow:/ User-agent: Spinn3r Disallow:/ User-agent: SputnikBot diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index a3458e1d5..31500b020 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2024.07.4650 -### Updated: Sat Jul 27 22:01:12 UTC 2024 -### Bad Bot Count: 665 +### Version: V4.2024.07.4651 +### Updated: Sun Jul 28 06:52:50 UTC 2024 +### Bad Bot Count: 667 ################################################### ### Version Information ## @@ -925,6 +925,10 @@ User-agent: Spanner Disallow:/ User-agent: Spbot Disallow:/ +User-agent: Spider_Bot +Disallow:/ +User-agent: Spider_Bot/3.0 +Disallow:/ User-agent: Spinn3r Disallow:/ User-agent: SputnikBot