diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index a6d76bb9f..7073e2b69 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2023.05.3716 +#### Version: V4.2023.05.3717 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [639](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [640](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 7e7d7313c..fdbd51ec1 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2023.05.3716 +#### Version: V4.2023.05.3717 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [639](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [640](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index 52ad6aae2..1214196b4 100755 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2023.05.3716 +#### Version: V4.2023.05.3717 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [639](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [640](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index ac994cb6d..800948f76 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2023.05.3716 +#### Version: V4.2023.05.3717 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [639](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [640](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index ced692e2e..21a18ec40 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3716 -### Updated: Mon May 1 04:23:14 UTC 2023 +### Version: V4.2023.05.3717 +### Updated: Mon May 1 04:37:16 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -553,6 +553,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Seekport(?:\b)" 3; + "~*(?:\b)SeekportBot(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index f790b02f0..1d2eb1b24 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2023.05.3716 -### Updated: Mon May 1 04:23:15 UTC 2023 +### Version: V3.2023.05.3717 +### Updated: Mon May 1 04:37:17 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -553,6 +553,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Seekport(?:\b)" 3; + "~*(?:\b)SeekportBot(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index ced692e2e..21a18ec40 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3716 -### Updated: Mon May 1 04:23:14 UTC 2023 +### Version: V4.2023.05.3717 +### Updated: Mon May 1 04:37:16 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -553,6 +553,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Seekport(?:\b)" 3; + "~*(?:\b)SeekportBot(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; diff --git a/dev-tools/_robots_input/robots-input.txt b/dev-tools/_robots_input/robots-input.txt index 627c3db0d..6b38a1b5f 100644 --- a/dev-tools/_robots_input/robots-input.txt +++ b/dev-tools/_robots_input/robots-input.txt @@ -404,6 +404,7 @@ ScrepyBot Searchestate SearchmetricsBot Seekport +SeekportBot SemanticJuice Semrush SemrushBot diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index afb7f113e..69e041273 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -3716 +3717 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index f790b02f0..1d2eb1b24 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2023.05.3716 -### Updated: Mon May 1 04:23:15 UTC 2023 +### Version: V3.2023.05.3717 +### Updated: Mon May 1 04:37:17 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -553,6 +553,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Seekport(?:\b)" 3; + "~*(?:\b)SeekportBot(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index ced692e2e..21a18ec40 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3716 -### Updated: Mon May 1 04:23:14 UTC 2023 +### Version: V4.2023.05.3717 +### Updated: Mon May 1 04:37:16 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -553,6 +553,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Seekport(?:\b)" 3; + "~*(?:\b)SeekportBot(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 0fee55db0..40955ca94 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -157.55.189.33 0; +40.86.28.128 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index ced692e2e..21a18ec40 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3716 -### Updated: Mon May 1 04:23:14 UTC 2023 +### Version: V4.2023.05.3717 +### Updated: Mon May 1 04:37:16 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -553,6 +553,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Seekport(?:\b)" 3; + "~*(?:\b)SeekportBot(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index d43a72e15..ee9141611 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.119 1; +10.1.0.44 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index a738ae780..9dcba65e6 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.119 0; +10.1.0.44 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index ced692e2e..21a18ec40 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3716 -### Updated: Mon May 1 04:23:14 UTC 2023 +### Version: V4.2023.05.3717 +### Updated: Mon May 1 04:37:16 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -553,6 +553,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Seekport(?:\b)" 3; + "~*(?:\b)SeekportBot(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index d43a72e15..ee9141611 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.119 1; +10.1.0.44 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index a738ae780..9dcba65e6 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.119 0; +10.1.0.44 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index ced692e2e..21a18ec40 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3716 -### Updated: Mon May 1 04:23:14 UTC 2023 +### Version: V4.2023.05.3717 +### Updated: Mon May 1 04:37:16 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -553,6 +553,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Seekport(?:\b)" 3; + "~*(?:\b)SeekportBot(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index d43a72e15..ee9141611 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.119 1; +10.1.0.44 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 25ccddb1c..dde25a039 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -465,6 +465,7 @@ "~*(?:\b)Searchestate(?:\b|)" 0; "~*(?:\b)SearchmetricsBot(?:\b|)" 0; "~*(?:\b)Seekport(?:\b|)" 0; + "~*(?:\b)SeekportBot(?:\b|)" 0; "~*(?:\b)SemanticJuice(?:\b|)" 0; "~*(?:\b)Semrush(?:\b|)" 0; "~*(?:\b)SemrushBot(?:\b|)" 0; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index a738ae780..9dcba65e6 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.119 0; +10.1.0.44 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index ced692e2e..21a18ec40 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3716 -### Updated: Mon May 1 04:23:14 UTC 2023 +### Version: V4.2023.05.3717 +### Updated: Mon May 1 04:37:16 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -553,6 +553,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Seekport(?:\b)" 3; + "~*(?:\b)SeekportBot(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 0fee55db0..40955ca94 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -157.55.189.33 0; +40.86.28.128 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index e85a7dbac..fc8fe99c4 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3716 -### Updated: Mon May 1 04:23:14 UTC 2023 +### Version: V4.2023.05.3717 +### Updated: Mon May 1 04:37:16 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -553,6 +553,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Seekport(?:\b)" 3; + "~*(?:\b)SeekportBot(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 0fee55db0..40955ca94 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -157.55.189.33 0; +40.86.28.128 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index bef7769d4..723fd89bc 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3716 -### Updated: Mon May 1 04:23:14 UTC 2023 +### Version: V4.2023.05.3717 +### Updated: Mon May 1 04:37:16 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -553,6 +553,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Searchestate(?:\b)" 3; "~*(?:\b)SearchmetricsBot(?:\b)" 3; "~*(?:\b)Seekport(?:\b)" 3; + "~*(?:\b)SeekportBot(?:\b)" 3; "~*(?:\b)SemanticJuice(?:\b)" 3; "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index d43a72e15..ee9141611 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.119 1; +10.1.0.44 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/blacklist-user-agents.conf b/dev-tools/test_units/blacklist-user-agents.conf index 25ccddb1c..dde25a039 100644 --- a/dev-tools/test_units/blacklist-user-agents.conf +++ b/dev-tools/test_units/blacklist-user-agents.conf @@ -465,6 +465,7 @@ "~*(?:\b)Searchestate(?:\b|)" 0; "~*(?:\b)SearchmetricsBot(?:\b|)" 0; "~*(?:\b)Seekport(?:\b|)" 0; + "~*(?:\b)SeekportBot(?:\b|)" 0; "~*(?:\b)SemanticJuice(?:\b|)" 0; "~*(?:\b)Semrush(?:\b|)" 0; "~*(?:\b)SemrushBot(?:\b|)" 0; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 8ded34e47..2df82da04 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -ALittle Client -Aspiegel -Bandit -Devil -FrontPage -Grabber -Kinza -Octopus -T8Abot -Telesphorep +CCBot +EyeNetIE +IDBTE4M +LinkWalker +MegaIndex.ru +Metauri +NetZIP +Titan +trendiction.com +voyagerx.com diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 1bbd104a7..461510c42 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,100 +1,100 @@ -404enemy -ADmantX -Acunetix -Anarchie -Ankit -Apexoo -BetaBot -Brandprotect -Buddy +AIBOT +Alligator +Anarchy +AwarioRssBot +BlackWidow +Boardreader CazoodleBot -ChinaClaw -Chlooe -Citoid -Claritybot -Devil -Discoverybot +CherryPicker +DBLBot +DatabaseDriverMysqli +Deusu +Dirbuster DnBCrawler-Analytics DomainAppender +EMail Wolf +EasyDL EirGrabber -Evil -ExtractorPro -FHscan +FDM +Firefox/7.0 GT::WWW -GetRight -GridBot +Getintent +Go!Zilla +Go-Ahead-Got-It +GoZilla +GrabNet +HEADMasterSEO HTMLparser -HybridBot -IlseBot -Image Fetch -InfoNaviRobot -InfoTekies -JamesBOT -JetCar -Keybot Translation-Search-Machine -LeechGet -Litemage_walker -Majestic-SEO -MarkMonitor -Meanpathbot -Mediatoolkitbot -Minefield +Humanlinks +Internet Ninja +Iskanie +Kinza +Larbin +LibWeb +LinkScan +Linkbot +LinkextractorPro +LinkpadBot +Lipperhey Spider +MQQBrowser Moblie Safari -Mr.4x3 -Nuclei -Offline Explorer +Mojeek +Nameprotect +Nessus +NetSpider +Nettrack +Octopus Offline Navigator -Openfind -Openvas -Panscient -Pavuk -PyCurl -RankActiveLinkBot -RankingBot -RankingBot2 -RankurBot -Reaper -RedesScrapy -ScoutJet -ScreenerBot +OpenLinkProfiler +OpenVAS +OutclicksBot +Page Analyzer +Papa Foto +Pi-Monster +PleaseCrawl +Pockey +Probethenet +Pu_iN +Recorder +RepoMonkey +Ripper +SEOkicks +ScanAlert +Scanbot ScrepyBot -SemanticJuice Semrush -SemrushBot -SentiBot -SiteLockSpider +Seomoz +Sitebeam Sociscraper +Steeler Sucker T8Abot -Toweyabot -Turingos -V-BOT +Telesphorep +Titan +Trendictionbot +VCI +Vacuum +Virusdie +WEBDAV WISENutbot -WWWOFFLE -WeSEE Web Enhancer +WebAuto WebCopier -WebReaper -Whatweb -Wotbox -Xaldon_WebSpider -YoudaoBot +WebImageCollector +WebWhacker +Website Quester +WebsiteExtractor +Xaldon WebSpider Zauba -ZoomBot ZoominfoBot archive.org_bot -backlink-check -com.plumanalytics +arquivo-web-crawler +gopher heritrix -instabid -internetVista monitor -isitwp.com -iubenda-radar -lwp-trivial -page scorer -plumanalytics -s1z.ru -sysscan -voyagerx.com -webmeup-crawler -x09Mozilla +netEstate NE Crawler +oBot +scan.lol +seobility +sexsearcher +spyfu +webgains-bot diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index a4235d549..41c74c983 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -groupmoney.ru -harmonyglen.us -letolove.ru -mosrif.ru -nootrino.com -picturesify.com -rome2rio.com -seorankinglinks.xyz -tanieaukcje.com.pl -zarabotat-na-sajte.ru +board.f00d.de +dnepr-avtospar.com.ua +favoritemoney.ru +kskjco.club +massage-info.nl +repeatlogo.co.uk +shoesonlinebuy.cn +waycash.net +wheelchairliftsarea.com +womensterritory.ru diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index 6cbd0aa37..56e7e3970 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,100 +1,100 @@ -100dollars-seo.com -123locker.com -1millionusd.xyz -300richmond.co.nz -3dracergames.com -4replicawatch.net -alta-realestate.com -am15.net -asdfz.pro -ayeartoforget.com -azartmsl.com -beztuberkuleza.ru -blavia.00author.com -boo-arts.com -brainsandeggs.blogspot.com -budpost.com.ua -cartiertrinity.xyz -cherrypointplace.ca -chorus.terakeet.com -christianlouboutinshoes.xyz -clickintext.net -commoncrawl.org -compliance-nelson.top -covadhosting.biz -covetnica.com -creativehutindia.com -dailyfinancefix.com -davebestdeals.com -deluxedumps.com -dividendo.ru -djekxa.ru -eaplay.ru -elektrischezi.canalblog.com -fastfixing.tech -fix-website-errors.com -friendflnder.com -fuckmill.com -gaz-voshod.ru -goldenggames.com -gwebtools.com.br -histock.info -inet-traffic.com -infazavr.ru -iplusbit.blogspot.co.za -iron-age.info -jbl-charge.info -johannesburgsingles.co.za -kopihijau.info -law-check-seven.xyz -lineavabit.it -lomza.info -losangeles-ads.com -makedo.ru -maladot.com -medicinacom.ru -ngps1.ru -nhl17coinsforps3.gratisblog.biz -openmediasoft.com -pic2fly.com -platesauto.com -prointer.net.ua -proposal-engine.com -prostitutki-belgoroda.org -reklama-i-rabota.ru -rockingclicks.com -rubbed.us -russintv.fr -sampleletters.net -scanner-don.top -scanner-jack.top -seorankinglinks.us -service.adtech.us -shop.xz618.com -sibdevice.ru -site3.free-share-buttons.com -sklad-24.ru -sovetogorod.ru -spitfiremusic.com -spywareit.com -stevemonsen.com -superoboi.com.ua -support.nopeas.sk -teenfuck.tv -telegraf.by -thebestweightlosspills.ovh -tootoo.to -topsiteminecraft.com -ultimateclassicrock.com -urengoy.pro -videojam.tv -wholesalejerseychinaoutlet.com -worldinternetauthority.com -x5market.ru -xn--c1acygb.xn--p1ai -xtubeporno.net -zahvat.ru -zaim-pod-zalog-krasnodar.ru -zigarettenonlinekaufen2.drupalgardens.com -zrelaya.pw -zs2vm.top +007angels.com +1kinobig.ru +7search.com +aa625d84f1587749c1ab011d6f269f7d64.com +aaronabel.com +accessoires-mode.in +acunetix-referrer.com +adman.gr +adprotect.net +agadelha.com.br +anniemation.com +apibrowsesmartne-a.akamaihd.net +app5.letmacworkfaster.world +appiq.mobi +atmovs.com +auto-moto-elektronika.cz +autoua.top +aviav.ru +babyguns.ru +blue-square.biz +bonzibuddi.com +buchananshardware.com +candycrushshop.com +cashonads.com +chastnoeporno.com +cheapestjerseysonwholesale.com +chelnytruck.ru +customcatchcan.com +detskie-konstruktory.ru +downloadme.life +e-c.al +edudocs.net +elektronischezigarettekaufen2.tumblr.com +eloconcream.blogspot.com +eropho.net +esopini.com +everflora.ru +eywords-monitoring-your-success.com +fapgon.com +floridamhca.org +gk170.ru +graphid.com +gsasearchengineranker.site +gsasearchengineranker.top +hamptonoaks.ca +hd720kino.ru +howmuchdoestizanidinecost.blogspot.com +iamsport.org +insider.pro +intellekt21.ru +iopeninghours.co.uk +kakadu-interior.com.ua +kinogonew.ru +konkursov.net +kosmetyki.tm.pl +kursy-ege.ru +leftofcentrist.com +limtu.ifmo.ru +linkrr.com +michaelkorsoutletstore.net +mp3ringtone.info +openmultipleurl.com +pihl.se +pizdeishn.com +porngalleries.top +prombudpostach.com.ua +promoheads.com +prostitutki-kharkova.org +pukaporn.com +qwesa.ru +ritlweb.com +running-line.ru +s-forum.biz +scanner-jessica.top +sex.hotblog.top +sexystrippe.info +sisi-go.ru +siteimprove.com +snomer1.ru +socialbuttons.xyz +soft-program.com +softwaretrend.net +solution4u.com +sport7777.net +strana-krasoty.ru +testbotprocessor44.com +tizanidinecapsules.blogspot.com +top10registrycleaners.com +trafficmp.com +undergroundcityphoto.com +valoresito.com +vapomnoncri.tk +versaut.xxx-cam.webcam +wait3sec.org +whatsappdownload10.com +xjrul.com +xlovecam.com +youtubologia.it +zeroredirect7.com +znakomstva-piter78.ru diff --git a/dev-tools/test_units/ratelimittest-master.txt b/dev-tools/test_units/ratelimittest-master.txt index 5580c125a..3656720f2 100644 --- a/dev-tools/test_units/ratelimittest-master.txt +++ b/dev-tools/test_units/ratelimittest-master.txt @@ -8,93 +8,6 @@

Welcome to the NginX Ultimate Bad Bot Blocker

This is merely used for testing !!!

-ml> -503 Service Temporarily Unavailable - -

503 Service Temporarily Unavailable

-
nginx/1.18.0 (Ubuntu)
- - - -503 Service Temporarily Unavailable - -

503 Service Temporarily Unavailable

-
nginx/1.18.0 (Ubuntu)
- - - -503 Service Temporarily Unavailable - -

503 Service Temporarily Unavailable

-
nginx/1.18.0 (Ubuntu)
- - - -503 Service Temporarily Unavailable - -

503 Service Temporarily Unavailable

-
nginx/1.18.0 (Ubuntu)
- - - -503 Service Temporarily Unavailable - -

503 Service Temporarily Unavailable

-
nginx/1.18.0 (Ubuntu)
- - - -503 Service Temporarily Unavailable - -

503 Service Temporarily Unavailable

-
nginx/1.18.0 (Ubuntu)
- - - -503 Service Temporarily Unavailable - -

503 Service Temporarily Unavailable

-
nginx/1.18.0 (Ubuntu)
- - - -503 Service Temporarily Unavailable - -

503 Service Temporarily Unavailable

-
nginx/1.18.0 (Ubuntu)
- - - -503 Service Temporarily Unavailable - -

503 Service Temporarily Unavailable

-
nginx/1.18.0 (Ubuntu)
- - - -503 Service Temporarily Unavailable - -

503 Service Temporarily Unavailable

-
nginx/1.18.0 (Ubuntu)
- - - -503 Service Temporarily Unavailable - -

503 Service Temporarily Unavailable

-
nginx/1.18.0 (Ubuntu)
- - - - - -Welcome to NginX - - -
-

Welcome to the NginX Ultimate Bad Bot Blocker

-

This is merely used for testing !!!

- 503 Service Temporarily Unavailable @@ -494,6 +407,90 @@
nginx/1.18.0 (Ubuntu)
+ +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.18.0 (Ubuntu)
+ + + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.18.0 (Ubuntu)
+ + + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.18.0 (Ubuntu)
+ + + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.18.0 (Ubuntu)
+ + + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.18.0 (Ubuntu)
+ + + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.18.0 (Ubuntu)
+ + + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.18.0 (Ubuntu)
+ + + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.18.0 (Ubuntu)
+ + + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.18.0 (Ubuntu)
+ + + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.18.0 (Ubuntu)
+ + + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.18.0 (Ubuntu)
+ + + +503 Service Temporarily Unavailable + +

503 Service Temporarily Unavailable

+
nginx/1.18.0 (Ubuntu)
+ + @@ -584,4 +581,14 @@

Welcome to the NginX Ultimate Bad Bot Blocker

This is merely used for testing !!!

+ + + +Welcome to NginX + + +
+

Welcome to the NginX Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ \ No newline at end of file diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index a738ae780..9dcba65e6 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.119 0; +10.1.0.44 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index 9f1fcb0d5..0982b2372 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2023.05.3716 -### Updated: Mon May 1 04:23:16 UTC 2023 -### Bad Bot Count: 639 +### Version: V4.2023.05.3717 +### Updated: Mon May 1 04:37:18 UTC 2023 +### Bad Bot Count: 640 ################################################### ### Version Information ## @@ -827,6 +827,8 @@ User-agent: SearchmetricsBot Disallow:/ User-agent: Seekport Disallow:/ +User-agent: SeekportBot +Disallow:/ User-agent: SemanticJuice Disallow:/ User-agent: Semrush diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 9f1fcb0d5..0982b2372 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2023.05.3716 -### Updated: Mon May 1 04:23:16 UTC 2023 -### Bad Bot Count: 639 +### Version: V4.2023.05.3717 +### Updated: Mon May 1 04:37:18 UTC 2023 +### Bad Bot Count: 640 ################################################### ### Version Information ## @@ -827,6 +827,8 @@ User-agent: SearchmetricsBot Disallow:/ User-agent: Seekport Disallow:/ +User-agent: SeekportBot +Disallow:/ User-agent: SemanticJuice Disallow:/ User-agent: Semrush