From 11ac00e2bbd5ae31415704ef3385602637202673 Mon Sep 17 00:00:00 2001 From: mitchellkrogza Date: Mon, 1 May 2023 06:04:59 +0000 Subject: [PATCH] V4.2023.05.3719 [ci skip] --- AUTO-CONFIGURATION.md | 4 +- MANUAL-CONFIGURATION.md | 4 +- README.md | 4 +- _generator_lists/bad-user-agents.list | 2 +- _google_analytics_ghost_spam/README.md | 4 +- .../etc/nginx/conf.d/globalblacklist.conf | 7 +- conf.d/globalblacklist-testing-version.conf | 7 +- conf.d/globalblacklist.conf | 7 +- dev-tools/_robots_input/robots-input.txt | 1 + dev-tools/buildnumber | 2 +- dev-tools/globalblacklist-testing.template | 7 +- dev-tools/globalblacklist.template | 7 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 7 +- .../conf.d/globalblacklist.conf | 7 +- .../conf.d/globalblacklist.conf | 7 +- .../bots.d/blacklist-user-agents.conf | 1 + .../conf.d/globalblacklist.conf | 7 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 7 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 7 +- .../test_units/blacklist-user-agents.conf | 1 + .../random-bots-for-test-quick.list | 20 +- .../random-bots-for-whitelist-test.list | 170 ++++++++-------- .../random-referrers-for-test-quick.list | 20 +- .../random-referrers-for-whitelist-test.list | 190 +++++++++--------- dev-tools/tmp/robots.txt | 8 +- robots.txt/robots.txt | 8 +- 29 files changed, 270 insertions(+), 252 deletions(-) diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 72accd693..2c086bf1e 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2023.05.3718 +#### Version: V4.2023.05.3719 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [639](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [640](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index c26cdb3ed..3f0641ebe 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2023.05.3718 +#### Version: V4.2023.05.3719 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [639](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [640](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index 0c3b5a4fc..f2b2f7c42 100755 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2023.05.3718 +#### Version: V4.2023.05.3719 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [639](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [640](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list index 1b66922fa..a705a18a9 100755 --- a/_generator_lists/bad-user-agents.list +++ b/_generator_lists/bad-user-agents.list @@ -1,4 +1,3 @@ -SenutoBot 01h4x.com 360Spider 404checker @@ -409,6 +408,7 @@ SemanticJuice Semrush SemrushBot SentiBot +SenutoBot SeoSiteCheckup SeobilityBot Seomoz diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 2d8de192c..169460aea 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2023.05.3718 +#### Version: V4.2023.05.3719 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [639](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [640](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 0924ed25a..681099a41 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3718 -### Updated: Mon May 1 05:50:03 UTC 2023 +### Version: V4.2023.05.3719 +### Updated: Mon May 1 06:01:10 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -557,6 +557,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SenutoBot(?:\b)" 3; "~*(?:\b)SeoSiteCheckup(?:\b)" 3; "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)Seomoz(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index df006d64f..fcd6fa9c2 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2023.05.3718 -### Updated: Mon May 1 05:50:04 UTC 2023 +### Version: V3.2023.05.3719 +### Updated: Mon May 1 06:01:11 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -557,6 +557,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SenutoBot(?:\b)" 3; "~*(?:\b)SeoSiteCheckup(?:\b)" 3; "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)Seomoz(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 0924ed25a..681099a41 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3718 -### Updated: Mon May 1 05:50:03 UTC 2023 +### Version: V4.2023.05.3719 +### Updated: Mon May 1 06:01:10 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -557,6 +557,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SenutoBot(?:\b)" 3; "~*(?:\b)SeoSiteCheckup(?:\b)" 3; "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)Seomoz(?:\b)" 3; diff --git a/dev-tools/_robots_input/robots-input.txt b/dev-tools/_robots_input/robots-input.txt index 5e0c4116f..a7f8acb33 100644 --- a/dev-tools/_robots_input/robots-input.txt +++ b/dev-tools/_robots_input/robots-input.txt @@ -408,6 +408,7 @@ SemanticJuice Semrush SemrushBot SentiBot +SenutoBot SeoSiteCheckup SeobilityBot Seomoz diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index 03d011a14..ea64d45ce 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -3718 +3719 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index df006d64f..fcd6fa9c2 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2023.05.3718 -### Updated: Mon May 1 05:50:04 UTC 2023 +### Version: V3.2023.05.3719 +### Updated: Mon May 1 06:01:11 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -557,6 +557,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SenutoBot(?:\b)" 3; "~*(?:\b)SeoSiteCheckup(?:\b)" 3; "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)Seomoz(?:\b)" 3; diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index 0924ed25a..681099a41 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3718 -### Updated: Mon May 1 05:50:03 UTC 2023 +### Version: V4.2023.05.3719 +### Updated: Mon May 1 06:01:10 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -557,6 +557,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SenutoBot(?:\b)" 3; "~*(?:\b)SeoSiteCheckup(?:\b)" 3; "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)Seomoz(?:\b)" 3; diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index c3f591b74..5dbbf6357 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.45.200.55 0; +40.75.119.48 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 0924ed25a..681099a41 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3718 -### Updated: Mon May 1 05:50:03 UTC 2023 +### Version: V4.2023.05.3719 +### Updated: Mon May 1 06:01:10 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -557,6 +557,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SenutoBot(?:\b)" 3; "~*(?:\b)SeoSiteCheckup(?:\b)" 3; "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)Seomoz(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 0924ed25a..681099a41 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3718 -### Updated: Mon May 1 05:50:03 UTC 2023 +### Version: V4.2023.05.3719 +### Updated: Mon May 1 06:01:10 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -557,6 +557,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SenutoBot(?:\b)" 3; "~*(?:\b)SeoSiteCheckup(?:\b)" 3; "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)Seomoz(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 0924ed25a..681099a41 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3718 -### Updated: Mon May 1 05:50:03 UTC 2023 +### Version: V4.2023.05.3719 +### Updated: Mon May 1 06:01:10 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -557,6 +557,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SenutoBot(?:\b)" 3; "~*(?:\b)SeoSiteCheckup(?:\b)" 3; "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)Seomoz(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 8af9f5370..eee5f2648 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -469,6 +469,7 @@ "~*(?:\b)Semrush(?:\b|)" 0; "~*(?:\b)SemrushBot(?:\b|)" 0; "~*(?:\b)SentiBot(?:\b|)" 0; + "~*(?:\b)SenutoBot(?:\b|)" 0; "~*(?:\b)SeoSiteCheckup(?:\b|)" 0; "~*(?:\b)SeobilityBot(?:\b|)" 0; "~*(?:\b)Seomoz(?:\b|)" 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 0924ed25a..681099a41 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3718 -### Updated: Mon May 1 05:50:03 UTC 2023 +### Version: V4.2023.05.3719 +### Updated: Mon May 1 06:01:10 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -557,6 +557,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SenutoBot(?:\b)" 3; "~*(?:\b)SeoSiteCheckup(?:\b)" 3; "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)Seomoz(?:\b)" 3; diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index c3f591b74..5dbbf6357 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.45.200.55 0; +40.75.119.48 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 6f16a8d71..7de976a95 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3718 -### Updated: Mon May 1 05:50:03 UTC 2023 +### Version: V4.2023.05.3719 +### Updated: Mon May 1 06:01:10 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -557,6 +557,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SenutoBot(?:\b)" 3; "~*(?:\b)SeoSiteCheckup(?:\b)" 3; "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)Seomoz(?:\b)" 3; diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index c3f591b74..5dbbf6357 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -104.45.200.55 0; +40.75.119.48 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index c8eb2647b..e35717b94 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.05.3718 -### Updated: Mon May 1 05:50:03 UTC 2023 +### Version: V4.2023.05.3719 +### Updated: Mon May 1 06:01:10 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 639 +### Bad Bot Count: 640 ################################################### ### VERSION INFORMATION ## @@ -557,6 +557,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Semrush(?:\b)" 3; "~*(?:\b)SemrushBot(?:\b)" 3; "~*(?:\b)SentiBot(?:\b)" 3; + "~*(?:\b)SenutoBot(?:\b)" 3; "~*(?:\b)SeoSiteCheckup(?:\b)" 3; "~*(?:\b)SeobilityBot(?:\b)" 3; "~*(?:\b)Seomoz(?:\b)" 3; diff --git a/dev-tools/test_units/blacklist-user-agents.conf b/dev-tools/test_units/blacklist-user-agents.conf index 8af9f5370..eee5f2648 100644 --- a/dev-tools/test_units/blacklist-user-agents.conf +++ b/dev-tools/test_units/blacklist-user-agents.conf @@ -469,6 +469,7 @@ "~*(?:\b)Semrush(?:\b|)" 0; "~*(?:\b)SemrushBot(?:\b|)" 0; "~*(?:\b)SentiBot(?:\b|)" 0; + "~*(?:\b)SenutoBot(?:\b|)" 0; "~*(?:\b)SeoSiteCheckup(?:\b|)" 0; "~*(?:\b)SeobilityBot(?:\b|)" 0; "~*(?:\b)Seomoz(?:\b|)" 0; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 576ffd200..ae2157722 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Black Hole -Go!Zilla -Heritrix -JetCar -Jorgee -LinkScan -PxBroker -SEOkicks -Thumbor -Whacker +404checker +JOC Web Spider +JikeSpider +Microsoft URL Control +SuperBot +Toata +Woobot +Wotbox +adscanner +isitwp.com diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 2191aa92d..5b4bcf92f 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,100 +1,100 @@ -404checker -Acunetix -AiHitBot -Aipbot +Aboundex AwarioSmartBot BBBike -BLEXBot -BackDoorBot +BDFetch +BackWeb +Badass +BatchFTP Battleztar Bazinga -Blackboard -BlowFish -BunnySlippers -CensysInspect -Citoid +CherryPicker +Cloud mapping Copier -Digincore -Dispatch -DittoSpyder -DomCopBot -DomainSigmaCrawler -Domains Project -Dragonfly -ECCP/1.0 +Copyscape +Crawling at Home Project +CrazyWebCrawler +CrunchBot +Custo +DataCha0s +DatabaseDriverMysqli +Dirbuster +DnyzBot +DomainCrawler Ebingbong -Exabot -ExtLinksBot -ExtractorPro +EirGrabber +Evil +Express WebPictures EyeNetIE -Fimap -Fuzz -GalaxyBot -Gotit -HTMLparser -Humanlinks -HybridBot -IlseBot +FDM +FHscan +Foobot +FrontPage +G-i-g-a-b-o-t +GrapeFX +HaosouSpider Image Fetch -IndeedBot -Indy Library -Likse -Lipperhey Spider -MSIECrawler +Image Sucker +InterGET +Jbrofuzz +LNSpiderguy +Lanshanbot +Linkbot +LinksManager +Ltx71 +Mag-Net Mail.RU_Bot -Masscan -Minefield -MolokaiBot -Morfeus Fucking Scanner -Nessus -Nibbler -Nikto +MarkMonitor +Microsoft Data Access +Moblie Safari +Nameprotect +NearSite Nimbostratus -Ninja -Nutch -OutclicksBot -PageScorer -Panscient -PxBroker -Quick-Crawler -Rankivabot -Rogerbot -SEOstats -SMTBot -ScrepyBot +Offline Navigator +Pavuk +Petalbot +Picsearch +ProWebWalker +RankurBot +Reaper +Searchestate Seomoz -Sitebeam -Sogou web spider -SpaceBison -SputnikBot +Sitevigil SuperBot -Suzuran -Telesphoreo -Telesphorep -TightTwatBot +Teleport Toweyabot -Voltron -WASALive-Bot +Trendictionbot +TwengaBot +Twice +Typhoeus +URLy.Warning +UnisterBot +Upflow +VCI +VelenPublicWebCrawler +VoidEYE +WEBDAV +WWW-Collector-E WWW::Mechanize -WallpapersHD -Web Sauger -Web Sucker -WebCollage -WebCopier -WebStripper -Whacker -Wotbox -Zitebot -ZmEu -ZoomBot -archive.org_bot -backlink-check -crawler.feedback -eCatch -heritrix -plumanalytics -probe-image-size -satoristudio.net -scan.lol +Web Auto +Web Collage +Web Fetch +WebSucker +Whack +Wprecon +Xaldon_WebSpider +Zade +Zauba +arquivo.pt +check1.exe +dataforseobot +domainsproject.org +lwp-request +page scorer +polaris version +s1z.ru +seobility +seoscanners seostar +sitechecker.pro spyfu -xpymep1.exe +trendiction.com +webpros.com diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 7d0984b7b..a64178341 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -3w24.com -focalink.com -landscapebackgrounds.blogspot.com -microstatic.pl -opinionreelle.com -petedrummond.com -petitions.whitehouse.gov -picturesmania.com -sexyali.com -tizanidinecost.blogspot.com +avon-ukraine.com +brateg.xyz +finemanteam.com +gambarkatabaru.com +hmywwogw.bloger.index.hr +jeremyeaton.co +modnie-futbolki.net +naphukete.ru +sudexpert66.ru +zaim-pod-zalog-krasnodar.ru diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index b4e263b2a..b40c867b4 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,100 +1,100 @@ -7makemoneyonline.com -aanapa.ru -airlandsea.info -all-streaming-media.com -allkrim.com +1688.com +1kdailyprofit.me +4istoshop.com +4ur.click +abwa.tk +adobereader-free.ru +adtech.fr +algarveglobal.com apimossnetinfo-a.akamaihd.net -apiwoodensealcom-a.akamaihd.net -bad-stars.net -balans.shahterworld.org -bestmarriages.com -bget.ru -bigpenisguide.com -braintobe.win -buckspar.gq -candlehandmade.com -caulderwoodkennels.com +arabseyes.com +azbukafree.com +bannerconnect.net +battleforupdating.site +beachtoday.ru +bestfortraders.com +bestplacetobuyeriacta.jw.lt +bestssaker.com +beta.hotkeys.com +bio-optomarket.ru +bioscorp.ru +blogs.rediff.com +bristolhotel.com +buchananshardware.com +candycrushshop.com +carstrends2015.com cercacamion.it -ci.ua +charmstroy.info cookingmeat.ru -credit-online.ws -dealighted.com -digital-sale.su -dolohen.com -domcran.net -domini.cat -dominterior.org -dumpsmania24.com -eloxal.ru -eu-cookie-law.blogspot.com -failingmarriege.blogspot.com -fortunejack.com -foto-sisek.porngalleries.top -frcls.fr -fsakhalin.ru -gsasearchengineranker.xyz -hulfingtonpost.com -igru-xbox.net -inspiring-desperate.tk -interesnie-faktu.ru -irkutsk.online-podarki.com -james13prix.info -jestr.org -konpax.com -krasula.pp.ua -linerdrilling.com -lordzfilmz.me -love-baby.cz -makenahartlin.com -moviezbonkerssk.cf -nbsproject.ru -nicovideo.jp -niroo.info -offf.info -okonich.com.ua -parvezmia.xyz -penisvergrotendepillennl.ovh -percin.biz.ly -pochemychka.net -polimga.pw -polska-poezja.com -popander.mobi -pyrodesigns.com.au -regdefense.com -ricorsogiustizia.org -seo-smm.kz -seocheckupx.com +covadhosting.biz +crynet.cc +d2jsp.org +demian.kz +devochki.top +dtnlyss.com +eas-seo.com +ebonyporn.site +enginebay.ru +ertelecom.ru +fashion-stickers.ru +fdzone.org +forum.doctissimo.fr +foxtechfpv.com +foxydeal.com +fr-bearings.ru +freshmac.space +fun-mobi.pl +gallictures.com +googst2.ru +gq-catalog.gq +hasfun.com +hdimagelib.com +healmytrauma.info +hmmm.cz +hobbyhuren-datenbank.com +hunterboots.online +hydropump.su +hyhj.info +infazavr.ru +iradiology.ru +liveinternet.ru +lost-alpha.ru +meds-online24.com +medtherapy.ru +mindbox.co.za +netoil.no +ntic.fr +onefilms.net +oranga.host.sk +pinkduck.ga +pinsdaddy.com +piter.xrus.org +piuminiita.com +plastweb.ru +pornotubexxx.name +prostitutki-tyumeni.org +questionmarque.ch +riralmolamsaca.tk +rossmark.ru +seomarketings.online +sex-dating.co sextracker.com -shop.xz618.com -sicfor.bcu.cc -social-s-iii.xyz +site3.free-share-buttons.com socialbookmarksubmission.org -solnplast.ru -stefanbakosab.se -thegolfclub.info -topappspro.com -topvidos.ru -trahvid.com -trumpetedextremes.com -tuberkuleznik.ru -twitlinks.com -tyumen.xrus.org -ukrobstep.com -unblocksit.es -uptime.com -usiad.net -video-chat.cn -videochat.tv.br -vip.51.la -vostoktrade.info -warning.or.kr -web-betting.ru -wnhjavlhezp.gq -woman-orgasm.ru -wttavern.com -xaylapdiendanang.com -xn----itbeirbjbi7bc6bh2d.xn--p1ai -zeg-distribution.com -zeroredirect9.com -zigarettenonlinekaufen2.bloog.pl -zytpirwai.net +sotkal.lark.ru +stauga.altervista.org +svensk-poesi.com +symphonyintegratedhealthcare.com +tabakur77.com +teenbbw.yopoint.in +themeforest.net +timer4web.com +tizanidinemusclerelaxant.blogspot.com +track112.site +treasuretrack-a.akamaihd.net +vapeface.club +vksaver-all.ru +wholesalejerseyscheapjerseys.us.com +wordpress-crew.net +yoshkarola.zrus.org +zrelaya.pw diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index b8923f14a..fc481efeb 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2023.05.3718 -### Updated: Mon May 1 05:50:05 UTC 2023 -### Bad Bot Count: 639 +### Version: V4.2023.05.3719 +### Updated: Mon May 1 06:01:11 UTC 2023 +### Bad Bot Count: 640 ################################################### ### Version Information ## @@ -835,6 +835,8 @@ User-agent: SemrushBot Disallow:/ User-agent: SentiBot Disallow:/ +User-agent: SenutoBot +Disallow:/ User-agent: SeoSiteCheckup Disallow:/ User-agent: SeobilityBot diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index b8923f14a..fc481efeb 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2023.05.3718 -### Updated: Mon May 1 05:50:05 UTC 2023 -### Bad Bot Count: 639 +### Version: V4.2023.05.3719 +### Updated: Mon May 1 06:01:11 UTC 2023 +### Bad Bot Count: 640 ################################################### ### Version Information ## @@ -835,6 +835,8 @@ User-agent: SemrushBot Disallow:/ User-agent: SentiBot Disallow:/ +User-agent: SenutoBot +Disallow:/ User-agent: SeoSiteCheckup Disallow:/ User-agent: SeobilityBot