diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index dfb76de2e..34328c0a0 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2023.10.4058 +#### Version: V4.2023.10.4059 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [645](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [647](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 655f1fce7..886b500bc 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2023.10.4058 +#### Version: V4.2023.10.4059 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [645](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [647](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index 629c40d6d..9ff3bd7d6 100755 --- a/README.md +++ b/README.md @@ -14,9 +14,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2023.10.4058 +#### Version: V4.2023.10.4059 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [645](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [647](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list index e51c97c38..af876deb4 100755 --- a/_generator_lists/bad-user-agents.list +++ b/_generator_lists/bad-user-agents.list @@ -369,6 +369,7 @@ PyCurl QueryN\ Metasearch Quick-Crawler RSSingBot +Rainbot RankActive RankActiveLinkBot RankFlex @@ -468,6 +469,7 @@ TheNomad The\ Intraformant Thumbor TightTwatBot +TinyTestBot Titan Toata Toweyabot @@ -643,5 +645,3 @@ x22Mozilla xpymep1.exe zauba.io zgrab -Rainbot -TinyTestBot diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index dc22f5583..8b28e915b 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2023.10.4058 +#### Version: V4.2023.10.4059 #### Bad Referrers Blocked: [7104](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [645](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [647](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 34fc9a3e0..726376633 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.10.4058 -### Updated: Mon Oct 16 10:03:34 UTC 2023 +### Version: V4.2023.10.4059 +### Updated: Mon Oct 16 11:26:14 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 645 +### Bad Bot Count: 647 ################################################### ### VERSION INFORMATION ## @@ -518,6 +518,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; "~*(?:\b)Quick-Crawler(?:\b)" 3; "~*(?:\b)RSSingBot(?:\b)" 3; + "~*(?:\b)Rainbot(?:\b)" 3; "~*(?:\b)RankActive(?:\b)" 3; "~*(?:\b)RankActiveLinkBot(?:\b)" 3; "~*(?:\b)RankFlex(?:\b)" 3; @@ -617,6 +618,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)The\ Intraformant(?:\b)" 3; "~*(?:\b)Thumbor(?:\b)" 3; "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)TinyTestBot(?:\b)" 3; "~*(?:\b)Titan(?:\b)" 3; "~*(?:\b)Toata(?:\b)" 3; "~*(?:\b)Toweyabot(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index d765a62d0..3a78d55c4 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2023.10.4058 -### Updated: Mon Oct 16 10:03:35 UTC 2023 +### Version: V3.2023.10.4059 +### Updated: Mon Oct 16 11:26:16 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 645 +### Bad Bot Count: 647 ################################################### ### VERSION INFORMATION ## @@ -518,6 +518,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; "~*(?:\b)Quick-Crawler(?:\b)" 3; "~*(?:\b)RSSingBot(?:\b)" 3; + "~*(?:\b)Rainbot(?:\b)" 3; "~*(?:\b)RankActive(?:\b)" 3; "~*(?:\b)RankActiveLinkBot(?:\b)" 3; "~*(?:\b)RankFlex(?:\b)" 3; @@ -617,6 +618,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)The\ Intraformant(?:\b)" 3; "~*(?:\b)Thumbor(?:\b)" 3; "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)TinyTestBot(?:\b)" 3; "~*(?:\b)Titan(?:\b)" 3; "~*(?:\b)Toata(?:\b)" 3; "~*(?:\b)Toweyabot(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 34fc9a3e0..726376633 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.10.4058 -### Updated: Mon Oct 16 10:03:34 UTC 2023 +### Version: V4.2023.10.4059 +### Updated: Mon Oct 16 11:26:14 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 645 +### Bad Bot Count: 647 ################################################### ### VERSION INFORMATION ## @@ -518,6 +518,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; "~*(?:\b)Quick-Crawler(?:\b)" 3; "~*(?:\b)RSSingBot(?:\b)" 3; + "~*(?:\b)Rainbot(?:\b)" 3; "~*(?:\b)RankActive(?:\b)" 3; "~*(?:\b)RankActiveLinkBot(?:\b)" 3; "~*(?:\b)RankFlex(?:\b)" 3; @@ -617,6 +618,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)The\ Intraformant(?:\b)" 3; "~*(?:\b)Thumbor(?:\b)" 3; "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)TinyTestBot(?:\b)" 3; "~*(?:\b)Titan(?:\b)" 3; "~*(?:\b)Toata(?:\b)" 3; "~*(?:\b)Toweyabot(?:\b)" 3; diff --git a/dev-tools/_robots_input/robots-input.txt b/dev-tools/_robots_input/robots-input.txt index a10a3d1dc..bc8beb518 100644 --- a/dev-tools/_robots_input/robots-input.txt +++ b/dev-tools/_robots_input/robots-input.txt @@ -369,6 +369,7 @@ PyCurl QueryN Metasearch Quick-Crawler RSSingBot +Rainbot RankActive RankActiveLinkBot RankFlex @@ -468,6 +469,7 @@ TheNomad The Intraformant Thumbor TightTwatBot +TinyTestBot Titan Toata Toweyabot diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index 30a03591e..76fa6727b 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -4058 +4059 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index d765a62d0..3a78d55c4 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2023.10.4058 -### Updated: Mon Oct 16 10:03:35 UTC 2023 +### Version: V3.2023.10.4059 +### Updated: Mon Oct 16 11:26:16 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 645 +### Bad Bot Count: 647 ################################################### ### VERSION INFORMATION ## @@ -518,6 +518,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; "~*(?:\b)Quick-Crawler(?:\b)" 3; "~*(?:\b)RSSingBot(?:\b)" 3; + "~*(?:\b)Rainbot(?:\b)" 3; "~*(?:\b)RankActive(?:\b)" 3; "~*(?:\b)RankActiveLinkBot(?:\b)" 3; "~*(?:\b)RankFlex(?:\b)" 3; @@ -617,6 +618,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)The\ Intraformant(?:\b)" 3; "~*(?:\b)Thumbor(?:\b)" 3; "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)TinyTestBot(?:\b)" 3; "~*(?:\b)Titan(?:\b)" 3; "~*(?:\b)Toata(?:\b)" 3; "~*(?:\b)Toweyabot(?:\b)" 3; diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index 34fc9a3e0..726376633 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.10.4058 -### Updated: Mon Oct 16 10:03:34 UTC 2023 +### Version: V4.2023.10.4059 +### Updated: Mon Oct 16 11:26:14 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 645 +### Bad Bot Count: 647 ################################################### ### VERSION INFORMATION ## @@ -518,6 +518,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; "~*(?:\b)Quick-Crawler(?:\b)" 3; "~*(?:\b)RSSingBot(?:\b)" 3; + "~*(?:\b)Rainbot(?:\b)" 3; "~*(?:\b)RankActive(?:\b)" 3; "~*(?:\b)RankActiveLinkBot(?:\b)" 3; "~*(?:\b)RankFlex(?:\b)" 3; @@ -617,6 +618,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)The\ Intraformant(?:\b)" 3; "~*(?:\b)Thumbor(?:\b)" 3; "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)TinyTestBot(?:\b)" 3; "~*(?:\b)Titan(?:\b)" 3; "~*(?:\b)Toata(?:\b)" 3; "~*(?:\b)Toweyabot(?:\b)" 3; diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index bf43616da..d701abdeb 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -40.84.173.36 0; +20.42.8.219 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 34fc9a3e0..726376633 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.10.4058 -### Updated: Mon Oct 16 10:03:34 UTC 2023 +### Version: V4.2023.10.4059 +### Updated: Mon Oct 16 11:26:14 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 645 +### Bad Bot Count: 647 ################################################### ### VERSION INFORMATION ## @@ -518,6 +518,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; "~*(?:\b)Quick-Crawler(?:\b)" 3; "~*(?:\b)RSSingBot(?:\b)" 3; + "~*(?:\b)Rainbot(?:\b)" 3; "~*(?:\b)RankActive(?:\b)" 3; "~*(?:\b)RankActiveLinkBot(?:\b)" 3; "~*(?:\b)RankFlex(?:\b)" 3; @@ -617,6 +618,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)The\ Intraformant(?:\b)" 3; "~*(?:\b)Thumbor(?:\b)" 3; "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)TinyTestBot(?:\b)" 3; "~*(?:\b)Titan(?:\b)" 3; "~*(?:\b)Toata(?:\b)" 3; "~*(?:\b)Toweyabot(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 4b0417f44..53409d87c 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.231 1; +10.1.0.16 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 1a7303ece..529aeb017 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.231 0; +10.1.0.16 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 34fc9a3e0..726376633 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.10.4058 -### Updated: Mon Oct 16 10:03:34 UTC 2023 +### Version: V4.2023.10.4059 +### Updated: Mon Oct 16 11:26:14 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 645 +### Bad Bot Count: 647 ################################################### ### VERSION INFORMATION ## @@ -518,6 +518,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; "~*(?:\b)Quick-Crawler(?:\b)" 3; "~*(?:\b)RSSingBot(?:\b)" 3; + "~*(?:\b)Rainbot(?:\b)" 3; "~*(?:\b)RankActive(?:\b)" 3; "~*(?:\b)RankActiveLinkBot(?:\b)" 3; "~*(?:\b)RankFlex(?:\b)" 3; @@ -617,6 +618,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)The\ Intraformant(?:\b)" 3; "~*(?:\b)Thumbor(?:\b)" 3; "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)TinyTestBot(?:\b)" 3; "~*(?:\b)Titan(?:\b)" 3; "~*(?:\b)Toata(?:\b)" 3; "~*(?:\b)Toweyabot(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 4b0417f44..53409d87c 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.231 1; +10.1.0.16 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 1a7303ece..529aeb017 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.231 0; +10.1.0.16 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 34fc9a3e0..726376633 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.10.4058 -### Updated: Mon Oct 16 10:03:34 UTC 2023 +### Version: V4.2023.10.4059 +### Updated: Mon Oct 16 11:26:14 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 645 +### Bad Bot Count: 647 ################################################### ### VERSION INFORMATION ## @@ -518,6 +518,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; "~*(?:\b)Quick-Crawler(?:\b)" 3; "~*(?:\b)RSSingBot(?:\b)" 3; + "~*(?:\b)Rainbot(?:\b)" 3; "~*(?:\b)RankActive(?:\b)" 3; "~*(?:\b)RankActiveLinkBot(?:\b)" 3; "~*(?:\b)RankFlex(?:\b)" 3; @@ -617,6 +618,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)The\ Intraformant(?:\b)" 3; "~*(?:\b)Thumbor(?:\b)" 3; "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)TinyTestBot(?:\b)" 3; "~*(?:\b)Titan(?:\b)" 3; "~*(?:\b)Toata(?:\b)" 3; "~*(?:\b)Toweyabot(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 4b0417f44..53409d87c 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.231 1; +10.1.0.16 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 27901341e..c52617f5e 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -430,6 +430,7 @@ "~*(?:\b)QueryN\ Metasearch(?:\b|)" 0; "~*(?:\b)Quick-Crawler(?:\b|)" 0; "~*(?:\b)RSSingBot(?:\b|)" 0; + "~*(?:\b)Rainbot(?:\b|)" 0; "~*(?:\b)RankActive(?:\b|)" 0; "~*(?:\b)RankActiveLinkBot(?:\b|)" 0; "~*(?:\b)RankFlex(?:\b|)" 0; @@ -529,6 +530,7 @@ "~*(?:\b)The\ Intraformant(?:\b|)" 0; "~*(?:\b)Thumbor(?:\b|)" 0; "~*(?:\b)TightTwatBot(?:\b|)" 0; + "~*(?:\b)TinyTestBot(?:\b|)" 0; "~*(?:\b)Titan(?:\b|)" 0; "~*(?:\b)Toata(?:\b|)" 0; "~*(?:\b)Toweyabot(?:\b|)" 0; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 1a7303ece..529aeb017 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.231 0; +10.1.0.16 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 34fc9a3e0..726376633 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.10.4058 -### Updated: Mon Oct 16 10:03:34 UTC 2023 +### Version: V4.2023.10.4059 +### Updated: Mon Oct 16 11:26:14 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 645 +### Bad Bot Count: 647 ################################################### ### VERSION INFORMATION ## @@ -518,6 +518,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; "~*(?:\b)Quick-Crawler(?:\b)" 3; "~*(?:\b)RSSingBot(?:\b)" 3; + "~*(?:\b)Rainbot(?:\b)" 3; "~*(?:\b)RankActive(?:\b)" 3; "~*(?:\b)RankActiveLinkBot(?:\b)" 3; "~*(?:\b)RankFlex(?:\b)" 3; @@ -617,6 +618,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)The\ Intraformant(?:\b)" 3; "~*(?:\b)Thumbor(?:\b)" 3; "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)TinyTestBot(?:\b)" 3; "~*(?:\b)Titan(?:\b)" 3; "~*(?:\b)Toata(?:\b)" 3; "~*(?:\b)Toweyabot(?:\b)" 3; diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index bf43616da..d701abdeb 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -40.84.173.36 0; +20.42.8.219 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 84b90e104..e9e5be1fb 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.10.4058 -### Updated: Mon Oct 16 10:03:34 UTC 2023 +### Version: V4.2023.10.4059 +### Updated: Mon Oct 16 11:26:14 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 645 +### Bad Bot Count: 647 ################################################### ### VERSION INFORMATION ## @@ -518,6 +518,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; "~*(?:\b)Quick-Crawler(?:\b)" 3; "~*(?:\b)RSSingBot(?:\b)" 3; + "~*(?:\b)Rainbot(?:\b)" 3; "~*(?:\b)RankActive(?:\b)" 3; "~*(?:\b)RankActiveLinkBot(?:\b)" 3; "~*(?:\b)RankFlex(?:\b)" 3; @@ -617,6 +618,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)The\ Intraformant(?:\b)" 3; "~*(?:\b)Thumbor(?:\b)" 3; "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)TinyTestBot(?:\b)" 3; "~*(?:\b)Titan(?:\b)" 3; "~*(?:\b)Toata(?:\b)" 3; "~*(?:\b)Toweyabot(?:\b)" 3; diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index bf43616da..d701abdeb 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -40.84.173.36 0; +20.42.8.219 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index d55cbfbcc..5223ce709 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2023.10.4058 -### Updated: Mon Oct 16 10:03:34 UTC 2023 +### Version: V4.2023.10.4059 +### Updated: Mon Oct 16 11:26:14 UTC 2023 ### Bad Referrer Count: 7104 -### Bad Bot Count: 645 +### Bad Bot Count: 647 ################################################### ### VERSION INFORMATION ## @@ -518,6 +518,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)QueryN\ Metasearch(?:\b)" 3; "~*(?:\b)Quick-Crawler(?:\b)" 3; "~*(?:\b)RSSingBot(?:\b)" 3; + "~*(?:\b)Rainbot(?:\b)" 3; "~*(?:\b)RankActive(?:\b)" 3; "~*(?:\b)RankActiveLinkBot(?:\b)" 3; "~*(?:\b)RankFlex(?:\b)" 3; @@ -617,6 +618,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)The\ Intraformant(?:\b)" 3; "~*(?:\b)Thumbor(?:\b)" 3; "~*(?:\b)TightTwatBot(?:\b)" 3; + "~*(?:\b)TinyTestBot(?:\b)" 3; "~*(?:\b)Titan(?:\b)" 3; "~*(?:\b)Toata(?:\b)" 3; "~*(?:\b)Toweyabot(?:\b)" 3; diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index 4b0417f44..53409d87c 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.231 1; +10.1.0.16 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/blacklist-user-agents.conf b/dev-tools/test_units/blacklist-user-agents.conf index 27901341e..c52617f5e 100644 --- a/dev-tools/test_units/blacklist-user-agents.conf +++ b/dev-tools/test_units/blacklist-user-agents.conf @@ -430,6 +430,7 @@ "~*(?:\b)QueryN\ Metasearch(?:\b|)" 0; "~*(?:\b)Quick-Crawler(?:\b|)" 0; "~*(?:\b)RSSingBot(?:\b|)" 0; + "~*(?:\b)Rainbot(?:\b|)" 0; "~*(?:\b)RankActive(?:\b|)" 0; "~*(?:\b)RankActiveLinkBot(?:\b|)" 0; "~*(?:\b)RankFlex(?:\b|)" 0; @@ -529,6 +530,7 @@ "~*(?:\b)The\ Intraformant(?:\b|)" 0; "~*(?:\b)Thumbor(?:\b|)" 0; "~*(?:\b)TightTwatBot(?:\b|)" 0; + "~*(?:\b)TinyTestBot(?:\b|)" 0; "~*(?:\b)Titan(?:\b|)" 0; "~*(?:\b)Toata(?:\b|)" 0; "~*(?:\b)Toweyabot(?:\b|)" 0; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 35896377e..97791dd90 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Bitacle -Demon -Jorgee -LeechFTP -Microsoft Data Access -Mojeek -SEOlyticsCrawler -WeSEE -Zade -demandbase-bot +BBBike +Blackboard +Citoid +Discobot +Kinza +Majestic-SEO +Mediatoolkitbot +Twice +backlink-check +sp_auditbot diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 832be5c25..a1fd31841 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,100 +1,100 @@ -80legs -ASPSeek -AdsTxtCrawlerTP +ALittle Client +Aboundexbot +AfD-Verbotsverfahren Alexibot -Asterias -Boardreader -Bolt -BotALot -Brandwatch -CheTeam -Cocolyzebot -Demon -Digincore +Anarchy +Ankit +AwarioRssBot +Badass +Barkrowler +BetaBot +Bitacle +Blow +CopyRightCheck +Curious +DTS Agent +Deusu +Devil DigitalPebble -Discobot -Dispatch -DnyzBot -DomainAppender -Download Wonder -EMail Wolf -EasyDL +DittoSpyder +DnBCrawler-Analytics +DomCopBot +DomainStatsBot Ecxi -Firefox/7.0 -Foobot -Go!Zilla -Go-Ahead-Got-It -GrapeshotCrawler -HEADMasterSEO -HTMLparser -HTTP::Lite -Havij -Iblog -IndeedBot -InternetSeer +Exabot +Ezooms +FemtosearchBot +Fimap +GT::WWW +GalaxyBot +Grabber +Haansoft +Harvest +InfoNaviRobot Iskanie -JustView -Keybot Translation-Search-Machine -Leap -LeechGet -LexiBot -LinkWalker -LinqiaMetadataDownloaderBot -LinqiaScrapeBot -Lmspider -MIDown tool -MSFrontPage -MSIECrawler -Mister PiX -MolokaiBot -Musobot -Needle -Netcraft -Nibbler -Page Analyzer -Papa Foto -Piepmatz -ProPowerBot +JikeSpider +Kinza +Lanshanbot +Libwhisker +Litemage_walker +Ltx71 +MQQBrowser +Mail.RU_Bot +Microsoft Data Access +Moblie Safari +NPbot +Nameprotect +Net Vampire +NetMechanic +Netvibes +Offline Explorer +Openvas +OrangeBot +Picscout +PictureFinder +Pimonster +Psbot RankActive -RankingBot -RankingBot2 ReGet SBIder -SeekportBot -SemanticJuice -SentiBot +SMTBot +ScanAlert +Scrapy +SearchmetricsBot +Seekport +SemrushBot Seomoz -Siteimprove -Sogou web spider -Sottopop -Steeler -SuperBot +Siphon +SiteCheckerBotCrawler +Spanner +Spinn3r +T0PHackTeam Teleport -Telesoft -TheNomad -Trendictionbot -Twice -Vacuum -Vagabondo -WBSearchBot -Web Auto -WebAuto -WebEnhancer -WebSucker -Website Quester -Webster -WinHTTrack -Wotbox -Xaldon WebSpider +TightTwatBot +Toata +TwengaBot +VCI +VelenPublicWebCrawler +VidibleScraper +Virusdie +WWW::Mechanize +WebsiteExtractor +Whatweb +Woobot Xenu +Zitebot +ZoominfoBot adscanner -arquivo.pt -eCatch -facebookscraper -lwp-request -meanpathbot +autoemailspider +heritrix +isitwp.com +openai.com +page scorer +pcBrowser +probe-image-size seoscanners seostar +trendiction.com +trendiction.de voyagerx.com -zauba.io zgrab diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index df07806c0..62a963501 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -101raccoon.ru -chatroulette.video -cialischmrx.com -event-tracking.com -gardene.ru -glopages.ru -letolove.ru -lomb.co -renecaovilla.online -snjatie-geroinovoy-lomki.ru +5000-cotydzien.com +buehne-fuer-menschenrechte.de +culturevie.info +dorratex.tn +go2album.com +howmuchdoestizanidinecost.blogspot.com +ipostroika.ru +medizinreisen.de +puserving.com +serw.clicksor.com diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index 0e96fc8a9..bdeca3554 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,100 +1,100 @@ -allboard.xobor.de -alloysteel.ru -arate.ru -avon-ukraine.com -backlinkwatch.com -bildsuche.ru -bio.trade-jp.net -blog.koorg.ru -camel-beach.com -cartierlove.xyz -cezartabac.ro -compliance-alex.xyz -compliance-brian.xyz -contextualyield.com -cordstrap.cc -culmimg.pw -customerguru.in -div.as -dolohen.com -dvervmoskvu.ru -e-commerce-seo.com -earncash.com.ua -evogarage.com -executehosting.com -expolicenciaslatam.co -ezofest.sk -f5mtrack.com -flashbannernow.com -fordsonmajbor.cf -forexgb.ru -forms-mtm.ru -gallery.rennlist.com -googst2.ru -goroda-vsego-mira.ru -gribokstop.com -ihtec2019.org -ilo134uloh.com -imperiafilm.ru -infospot.pt -innodgfdriverhm.aircus.com -inome.com.ua -iskalko.ru -itbc.kiev.ua -jikoman.info -kaac.ru -kinostorm.net -kvartira-sutochno.com -landmania.ru -letsart.ru -limads.men +45en.ru +999webdesign.com +adloads.com +aramaicmedia.org +aversis.net +aviav.co +bannerbank.ru +beclean-nn.ru +bidbuy.co.kr +bonzibuddi.com +cialischmrx.com +commoncrawl.org +cornomase.win +corta.co +creatives.adbetclickin.pink +dev.citetick.com +donna7753191.ru +elvenmachine.com +englishgamer.com +erolate.com +erotik0049.com +etotupo.ru +etur.ru +exdocsfiles.com +ezigarettenshop1.yolasite.com +fba-mexico.com +filesdatabase.net +footbalive.org +freshsuperbloop.com +freza-sverlo.ru +frvo.alptandem.ru +ftns.ru +fyxabomiw.ru +gearsadspromo.club +getmiro.com +grtyi.com +hdpixent.com +hobbyhuren-datenbank.com +houseofrose.com +ideibiznesa2015.ru +ilte.info +infoupdate.org +intervsem.ru +ispac.org +jobmarket.com.ua +kinoflux.net +kinozapas.org +krasnodar-avtolombards.ru +landscaping.center +lexixxx.com +lion.playtap.us +locustdesign.co +mafcards.ru +medicinacom.ru meetlocalchicks.com -mesbuta.info -metallosajding.ru -minet.club -my-floor.in.ua -naturtreenspicerx.pw -no-fuel.org -noscrapleftbehind.co -onclkads.com -pcgroup.com.uy -pharmacyincity.com -play.leadzupc.com -popup-fdm.xyz -pozdrawleniya.ru -producm.ru -prospekt-st.ru -purplesphere.in -rada.ru +mega-bony-2017.pl +methodsmarketing.com +mielec.pl +mojpregled.com +moncleroutletonline.win +money-every-day.com +mycouponizemac.com +myra.top +onlainbesplatno.ru +pendelprognos.se +photostudiolightings.com +pobeiranie.pl +porndairy.in +pornokajf.com +promodj.com razamicroelectronics.com -redhotfreebies.co.uk -rep-am.com -replicalouboutin.xyz -ringtonepartner.com -sale-japan.com -seogadget.ru -sicfor.bcu.cc -soft-program.com -stretchingabuckblog.com -supervesti.ru -thecoral.com.br -thefds.net -thegameriders.com -tizanidinedrugtest.blogspot.com -tovaroboom.vast.ru -tsyndicate.com -txxx.com -videochat.cafe -vip-dom.in -vkontarkte.com -vykup-avto-krasnodar.ru -vzubah.com -watchmygf.net -websitevaluebot.com -womama.ru -xcombear.ru -xn----btbdvdh4aafrfciljm6k.xn--p1ai -xxxtubesafari.com -yes-com.com -zaobao.com.sg -zx6.ru +rebuildermedical.com +redirectingat.com +referencemoi.com +resant.ru +revistaindustria.com +sbf441.com +scanner-jess.top +scanner-willy.top +sexblog.pw +sexsaoy.com +sms2x2.ru +solinf.co +speedup-my.site +spylog.com +stauga.altervista.org +sureone.pro +symbaloo.com +taxi-v-eisk.ru +topflownews.com +v720hd.ru +videochat.ph +videochat.tv.br +viteonlusarezzo.it +vzlomfb.com +webradiology.ru +www.souvenirua.com +xn--80ajbshivpvn2i.xn--p1ai +yurgorod.ru +zooggames.com diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index 1a7303ece..529aeb017 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.231 0; +10.1.0.16 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index cc85a8e82..c41830d03 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2023.10.4058 -### Updated: Mon Oct 16 10:03:36 UTC 2023 -### Bad Bot Count: 645 +### Version: V4.2023.10.4059 +### Updated: Mon Oct 16 11:26:16 UTC 2023 +### Bad Bot Count: 647 ################################################### ### Version Information ## @@ -757,6 +757,8 @@ User-agent: Quick-Crawler Disallow:/ User-agent: RSSingBot Disallow:/ +User-agent: Rainbot +Disallow:/ User-agent: RankActive Disallow:/ User-agent: RankActiveLinkBot @@ -955,6 +957,8 @@ User-agent: Thumbor Disallow:/ User-agent: TightTwatBot Disallow:/ +User-agent: TinyTestBot +Disallow:/ User-agent: Titan Disallow:/ User-agent: Toata diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index cc85a8e82..c41830d03 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2023.10.4058 -### Updated: Mon Oct 16 10:03:36 UTC 2023 -### Bad Bot Count: 645 +### Version: V4.2023.10.4059 +### Updated: Mon Oct 16 11:26:16 UTC 2023 +### Bad Bot Count: 647 ################################################### ### Version Information ## @@ -757,6 +757,8 @@ User-agent: Quick-Crawler Disallow:/ User-agent: RSSingBot Disallow:/ +User-agent: Rainbot +Disallow:/ User-agent: RankActive Disallow:/ User-agent: RankActiveLinkBot @@ -955,6 +957,8 @@ User-agent: Thumbor Disallow:/ User-agent: TightTwatBot Disallow:/ +User-agent: TinyTestBot +Disallow:/ User-agent: Titan Disallow:/ User-agent: Toata