diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index d34dc167a..fc82526aa 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.08.2510 +#### Version: V4.2021.08.2511 #### Bad Referrers Blocked: [7079](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [618](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [620](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 97220287c..da6afc94c 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.08.2510 +#### Version: V4.2021.08.2511 #### Bad Referrers Blocked: [7079](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [618](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [620](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index 61d6e5c47..234651a24 100755 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2021.08.2510 +#### Version: V4.2021.08.2511 #### Bad Referrers Blocked: [7079](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [618](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [620](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list index b75c70e03..939fd2a99 100755 --- a/_generator_lists/bad-user-agents.list +++ b/_generator_lists/bad-user-agents.list @@ -1,5 +1,3 @@ -cah.io.community -Crawling\ at\ Home\ Project 01h4x.com 360Spider 404checker @@ -90,6 +88,7 @@ CopyRightCheck Copyscape Cosmos Craftbot +Crawling\ at\ Home\ Project CrazyWebCrawler Crescent CrunchBot @@ -560,6 +559,7 @@ arquivo-web-crawler arquivo.pt autoemailspider backlink-check +cah.io.community check1.exe coccocbot-web cognitiveseo diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 600481ae6..53c6c8139 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2021.08.2510 +#### Version: V4.2021.08.2511 #### Bad Referrers Blocked: [7079](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [618](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [620](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 60a961b4c..04e306a71 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.08.2510 -### Updated: Sun Aug 8 08:39:07 UTC 2021 +### Version: V4.2021.08.2511 +### Updated: Sun Aug 8 09:36:42 UTC 2021 ### Bad Referrer Count: 7079 -### Bad Bot Count: 618 +### Bad Bot Count: 620 ################################################### ### VERSION INFORMATION ## @@ -237,6 +237,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Copyscape(?:\b)" 3; "~*(?:\b)Cosmos(?:\b)" 3; "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)Crawling\ at\ Home\ Project(?:\b)" 3; "~*(?:\b)CrazyWebCrawler(?:\b)" 3; "~*(?:\b)Crescent(?:\b)" 3; "~*(?:\b)CrunchBot(?:\b)" 3; @@ -707,6 +708,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 326527154..eedfd78c4 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.08.2510 -### Updated: Sun Aug 8 08:39:08 UTC 2021 +### Version: V3.2021.08.2511 +### Updated: Sun Aug 8 09:36:42 UTC 2021 ### Bad Referrer Count: 7079 -### Bad Bot Count: 618 +### Bad Bot Count: 620 ################################################### ### VERSION INFORMATION ## @@ -237,6 +237,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Copyscape(?:\b)" 3; "~*(?:\b)Cosmos(?:\b)" 3; "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)Crawling\ at\ Home\ Project(?:\b)" 3; "~*(?:\b)CrazyWebCrawler(?:\b)" 3; "~*(?:\b)Crescent(?:\b)" 3; "~*(?:\b)CrunchBot(?:\b)" 3; @@ -707,6 +708,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 60a961b4c..04e306a71 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.08.2510 -### Updated: Sun Aug 8 08:39:07 UTC 2021 +### Version: V4.2021.08.2511 +### Updated: Sun Aug 8 09:36:42 UTC 2021 ### Bad Referrer Count: 7079 -### Bad Bot Count: 618 +### Bad Bot Count: 620 ################################################### ### VERSION INFORMATION ## @@ -237,6 +237,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Copyscape(?:\b)" 3; "~*(?:\b)Cosmos(?:\b)" 3; "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)Crawling\ at\ Home\ Project(?:\b)" 3; "~*(?:\b)CrazyWebCrawler(?:\b)" 3; "~*(?:\b)Crescent(?:\b)" 3; "~*(?:\b)CrunchBot(?:\b)" 3; @@ -707,6 +708,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; diff --git a/dev-tools/_robots_input/robots-input.txt b/dev-tools/_robots_input/robots-input.txt index df43911b1..cc1eca95e 100644 --- a/dev-tools/_robots_input/robots-input.txt +++ b/dev-tools/_robots_input/robots-input.txt @@ -88,6 +88,7 @@ CopyRightCheck Copyscape Cosmos Craftbot +Crawling at Home Project CrazyWebCrawler Crescent CrunchBot @@ -558,6 +559,7 @@ arquivo-web-crawler arquivo.pt autoemailspider backlink-check +cah.io.community check1.exe coccocbot-web cognitiveseo diff --git a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 326527154..eedfd78c4 100644 --- a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.08.2510 -### Updated: Sun Aug 8 08:39:08 UTC 2021 +### Version: V3.2021.08.2511 +### Updated: Sun Aug 8 09:36:42 UTC 2021 ### Bad Referrer Count: 7079 -### Bad Bot Count: 618 +### Bad Bot Count: 620 ################################################### ### VERSION INFORMATION ## @@ -237,6 +237,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Copyscape(?:\b)" 3; "~*(?:\b)Cosmos(?:\b)" 3; "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)Crawling\ at\ Home\ Project(?:\b)" 3; "~*(?:\b)CrazyWebCrawler(?:\b)" 3; "~*(?:\b)Crescent(?:\b)" 3; "~*(?:\b)CrunchBot(?:\b)" 3; @@ -707,6 +708,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index ad2c5c67f..b21bea8dc 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.228 1; +10.1.0.47 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index f1c3ae34d..f3b3bcf9d 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.228 0; +10.1.0.47 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 326527154..eedfd78c4 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.08.2510 -### Updated: Sun Aug 8 08:39:08 UTC 2021 +### Version: V3.2021.08.2511 +### Updated: Sun Aug 8 09:36:42 UTC 2021 ### Bad Referrer Count: 7079 -### Bad Bot Count: 618 +### Bad Bot Count: 620 ################################################### ### VERSION INFORMATION ## @@ -237,6 +237,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Copyscape(?:\b)" 3; "~*(?:\b)Cosmos(?:\b)" 3; "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)Crawling\ at\ Home\ Project(?:\b)" 3; "~*(?:\b)CrazyWebCrawler(?:\b)" 3; "~*(?:\b)Crescent(?:\b)" 3; "~*(?:\b)CrunchBot(?:\b)" 3; @@ -707,6 +708,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index ad2c5c67f..b21bea8dc 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.228 1; +10.1.0.47 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index f1c3ae34d..f3b3bcf9d 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.228 0; +10.1.0.47 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 326527154..eedfd78c4 100644 --- a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.08.2510 -### Updated: Sun Aug 8 08:39:08 UTC 2021 +### Version: V3.2021.08.2511 +### Updated: Sun Aug 8 09:36:42 UTC 2021 ### Bad Referrer Count: 7079 -### Bad Bot Count: 618 +### Bad Bot Count: 620 ################################################### ### VERSION INFORMATION ## @@ -237,6 +237,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Copyscape(?:\b)" 3; "~*(?:\b)Cosmos(?:\b)" 3; "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)Crawling\ at\ Home\ Project(?:\b)" 3; "~*(?:\b)CrazyWebCrawler(?:\b)" 3; "~*(?:\b)Crescent(?:\b)" 3; "~*(?:\b)CrunchBot(?:\b)" 3; @@ -707,6 +708,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index ad2c5c67f..b21bea8dc 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.228 1; +10.1.0.47 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf index f6a03bb56..8d5c0e24a 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -149,6 +149,7 @@ "~*(?:\b)Copyscape(?:\b|)" 0; "~*(?:\b)Cosmos(?:\b|)" 0; "~*(?:\b)Craftbot(?:\b|)" 0; + "~*(?:\b)Crawling\ at\ Home\ Project(?:\b|)" 0; "~*(?:\b)CrazyWebCrawler(?:\b|)" 0; "~*(?:\b)Crescent(?:\b|)" 0; "~*(?:\b)CrunchBot(?:\b|)" 0; @@ -619,6 +620,7 @@ "~*(?:\b)arquivo.pt(?:\b|)" 0; "~*(?:\b)autoemailspider(?:\b|)" 0; "~*(?:\b)backlink-check(?:\b|)" 0; + "~*(?:\b)cah.io.community(?:\b|)" 0; "~*(?:\b)check1.exe(?:\b|)" 0; "~*(?:\b)coccocbot-web(?:\b|)" 0; "~*(?:\b)cognitiveseo(?:\b|)" 0; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index f1c3ae34d..f3b3bcf9d 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.228 0; +10.1.0.47 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 326527154..eedfd78c4 100644 --- a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.08.2510 -### Updated: Sun Aug 8 08:39:08 UTC 2021 +### Version: V3.2021.08.2511 +### Updated: Sun Aug 8 09:36:42 UTC 2021 ### Bad Referrer Count: 7079 -### Bad Bot Count: 618 +### Bad Bot Count: 620 ################################################### ### VERSION INFORMATION ## @@ -237,6 +237,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Copyscape(?:\b)" 3; "~*(?:\b)Cosmos(?:\b)" 3; "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)Crawling\ at\ Home\ Project(?:\b)" 3; "~*(?:\b)CrazyWebCrawler(?:\b)" 3; "~*(?:\b)Crescent(?:\b)" 3; "~*(?:\b)CrunchBot(?:\b)" 3; @@ -707,6 +708,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index d0f0d290c..cc52cd507 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -2510 +2511 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index 326527154..eedfd78c4 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.08.2510 -### Updated: Sun Aug 8 08:39:08 UTC 2021 +### Version: V3.2021.08.2511 +### Updated: Sun Aug 8 09:36:42 UTC 2021 ### Bad Referrer Count: 7079 -### Bad Bot Count: 618 +### Bad Bot Count: 620 ################################################### ### VERSION INFORMATION ## @@ -237,6 +237,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Copyscape(?:\b)" 3; "~*(?:\b)Cosmos(?:\b)" 3; "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)Crawling\ at\ Home\ Project(?:\b)" 3; "~*(?:\b)CrazyWebCrawler(?:\b)" 3; "~*(?:\b)Crescent(?:\b)" 3; "~*(?:\b)CrunchBot(?:\b)" 3; @@ -707,6 +708,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index 60a961b4c..04e306a71 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.08.2510 -### Updated: Sun Aug 8 08:39:07 UTC 2021 +### Version: V4.2021.08.2511 +### Updated: Sun Aug 8 09:36:42 UTC 2021 ### Bad Referrer Count: 7079 -### Bad Bot Count: 618 +### Bad Bot Count: 620 ################################################### ### VERSION INFORMATION ## @@ -237,6 +237,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Copyscape(?:\b)" 3; "~*(?:\b)Cosmos(?:\b)" 3; "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)Crawling\ at\ Home\ Project(?:\b)" 3; "~*(?:\b)CrazyWebCrawler(?:\b)" 3; "~*(?:\b)Crescent(?:\b)" 3; "~*(?:\b)CrunchBot(?:\b)" 3; @@ -707,6 +708,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 7a5394900..beb7628d8 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -40.121.235.86 0; +40.71.167.189 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 60a961b4c..04e306a71 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.08.2510 -### Updated: Sun Aug 8 08:39:07 UTC 2021 +### Version: V4.2021.08.2511 +### Updated: Sun Aug 8 09:36:42 UTC 2021 ### Bad Referrer Count: 7079 -### Bad Bot Count: 618 +### Bad Bot Count: 620 ################################################### ### VERSION INFORMATION ## @@ -237,6 +237,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Copyscape(?:\b)" 3; "~*(?:\b)Cosmos(?:\b)" 3; "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)Crawling\ at\ Home\ Project(?:\b)" 3; "~*(?:\b)CrazyWebCrawler(?:\b)" 3; "~*(?:\b)Crescent(?:\b)" 3; "~*(?:\b)CrunchBot(?:\b)" 3; @@ -707,6 +708,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index ad2c5c67f..b21bea8dc 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.228 1; +10.1.0.47 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index f1c3ae34d..f3b3bcf9d 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.228 0; +10.1.0.47 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 60a961b4c..04e306a71 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.08.2510 -### Updated: Sun Aug 8 08:39:07 UTC 2021 +### Version: V4.2021.08.2511 +### Updated: Sun Aug 8 09:36:42 UTC 2021 ### Bad Referrer Count: 7079 -### Bad Bot Count: 618 +### Bad Bot Count: 620 ################################################### ### VERSION INFORMATION ## @@ -237,6 +237,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Copyscape(?:\b)" 3; "~*(?:\b)Cosmos(?:\b)" 3; "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)Crawling\ at\ Home\ Project(?:\b)" 3; "~*(?:\b)CrazyWebCrawler(?:\b)" 3; "~*(?:\b)Crescent(?:\b)" 3; "~*(?:\b)CrunchBot(?:\b)" 3; @@ -707,6 +708,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index ad2c5c67f..b21bea8dc 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.228 1; +10.1.0.47 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index f1c3ae34d..f3b3bcf9d 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.228 0; +10.1.0.47 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 60a961b4c..04e306a71 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.08.2510 -### Updated: Sun Aug 8 08:39:07 UTC 2021 +### Version: V4.2021.08.2511 +### Updated: Sun Aug 8 09:36:42 UTC 2021 ### Bad Referrer Count: 7079 -### Bad Bot Count: 618 +### Bad Bot Count: 620 ################################################### ### VERSION INFORMATION ## @@ -237,6 +237,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Copyscape(?:\b)" 3; "~*(?:\b)Cosmos(?:\b)" 3; "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)Crawling\ at\ Home\ Project(?:\b)" 3; "~*(?:\b)CrazyWebCrawler(?:\b)" 3; "~*(?:\b)Crescent(?:\b)" 3; "~*(?:\b)CrunchBot(?:\b)" 3; @@ -707,6 +708,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index ad2c5c67f..b21bea8dc 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.228 1; +10.1.0.47 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index f6a03bb56..8d5c0e24a 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -149,6 +149,7 @@ "~*(?:\b)Copyscape(?:\b|)" 0; "~*(?:\b)Cosmos(?:\b|)" 0; "~*(?:\b)Craftbot(?:\b|)" 0; + "~*(?:\b)Crawling\ at\ Home\ Project(?:\b|)" 0; "~*(?:\b)CrazyWebCrawler(?:\b|)" 0; "~*(?:\b)Crescent(?:\b|)" 0; "~*(?:\b)CrunchBot(?:\b|)" 0; @@ -619,6 +620,7 @@ "~*(?:\b)arquivo.pt(?:\b|)" 0; "~*(?:\b)autoemailspider(?:\b|)" 0; "~*(?:\b)backlink-check(?:\b|)" 0; + "~*(?:\b)cah.io.community(?:\b|)" 0; "~*(?:\b)check1.exe(?:\b|)" 0; "~*(?:\b)coccocbot-web(?:\b|)" 0; "~*(?:\b)cognitiveseo(?:\b|)" 0; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index f1c3ae34d..f3b3bcf9d 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.228 0; +10.1.0.47 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 60a961b4c..04e306a71 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.08.2510 -### Updated: Sun Aug 8 08:39:07 UTC 2021 +### Version: V4.2021.08.2511 +### Updated: Sun Aug 8 09:36:42 UTC 2021 ### Bad Referrer Count: 7079 -### Bad Bot Count: 618 +### Bad Bot Count: 620 ################################################### ### VERSION INFORMATION ## @@ -237,6 +237,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Copyscape(?:\b)" 3; "~*(?:\b)Cosmos(?:\b)" 3; "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)Crawling\ at\ Home\ Project(?:\b)" 3; "~*(?:\b)CrazyWebCrawler(?:\b)" 3; "~*(?:\b)Crescent(?:\b)" 3; "~*(?:\b)CrunchBot(?:\b)" 3; @@ -707,6 +708,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 7a5394900..beb7628d8 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -40.121.235.86 0; +40.71.167.189 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 2265cb144..3a624b4a2 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.08.2510 -### Updated: Sun Aug 8 08:39:07 UTC 2021 +### Version: V4.2021.08.2511 +### Updated: Sun Aug 8 09:36:42 UTC 2021 ### Bad Referrer Count: 7079 -### Bad Bot Count: 618 +### Bad Bot Count: 620 ################################################### ### VERSION INFORMATION ## @@ -237,6 +237,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Copyscape(?:\b)" 3; "~*(?:\b)Cosmos(?:\b)" 3; "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)Crawling\ at\ Home\ Project(?:\b)" 3; "~*(?:\b)CrazyWebCrawler(?:\b)" 3; "~*(?:\b)Crescent(?:\b)" 3; "~*(?:\b)CrunchBot(?:\b)" 3; @@ -707,6 +708,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 7a5394900..beb7628d8 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -40.121.235.86 0; +40.71.167.189 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 0e5c7b915..f8de16a18 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.08.2510 -### Updated: Sun Aug 8 08:39:07 UTC 2021 +### Version: V4.2021.08.2511 +### Updated: Sun Aug 8 09:36:42 UTC 2021 ### Bad Referrer Count: 7079 -### Bad Bot Count: 618 +### Bad Bot Count: 620 ################################################### ### VERSION INFORMATION ## @@ -237,6 +237,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)Copyscape(?:\b)" 3; "~*(?:\b)Cosmos(?:\b)" 3; "~*(?:\b)Craftbot(?:\b)" 3; + "~*(?:\b)Crawling\ at\ Home\ Project(?:\b)" 3; "~*(?:\b)CrazyWebCrawler(?:\b)" 3; "~*(?:\b)Crescent(?:\b)" 3; "~*(?:\b)CrunchBot(?:\b)" 3; @@ -707,6 +708,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)arquivo.pt(?:\b)" 3; "~*(?:\b)autoemailspider(?:\b)" 3; "~*(?:\b)backlink-check(?:\b)" 3; + "~*(?:\b)cah.io.community(?:\b)" 3; "~*(?:\b)check1.exe(?:\b)" 3; "~*(?:\b)coccocbot-web(?:\b)" 3; "~*(?:\b)cognitiveseo(?:\b)" 3; diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index ad2c5c67f..b21bea8dc 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.228 1; +10.1.0.47 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/blacklist-user-agents.conf b/dev-tools/test_units/blacklist-user-agents.conf index f6a03bb56..8d5c0e24a 100644 --- a/dev-tools/test_units/blacklist-user-agents.conf +++ b/dev-tools/test_units/blacklist-user-agents.conf @@ -149,6 +149,7 @@ "~*(?:\b)Copyscape(?:\b|)" 0; "~*(?:\b)Cosmos(?:\b|)" 0; "~*(?:\b)Craftbot(?:\b|)" 0; + "~*(?:\b)Crawling\ at\ Home\ Project(?:\b|)" 0; "~*(?:\b)CrazyWebCrawler(?:\b|)" 0; "~*(?:\b)Crescent(?:\b|)" 0; "~*(?:\b)CrunchBot(?:\b|)" 0; @@ -619,6 +620,7 @@ "~*(?:\b)arquivo.pt(?:\b|)" 0; "~*(?:\b)autoemailspider(?:\b|)" 0; "~*(?:\b)backlink-check(?:\b|)" 0; + "~*(?:\b)cah.io.community(?:\b|)" 0; "~*(?:\b)check1.exe(?:\b|)" 0; "~*(?:\b)coccocbot-web(?:\b|)" 0; "~*(?:\b)cognitiveseo(?:\b|)" 0; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 5adb80678..696c250f1 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Joomla -LibWeb -MarkWatch -Mojolicious -OrangeSpider -PECL::HTTP -Telesphorep -Upflow -ZmEu -eCatch +Abonti +Brandprotect +DatabaseDriverMysqli +FDM +Offline Explorer +OpenVAS +Re-re +SemrushBot +SuperHTTP +siteripz diff --git a/dev-tools/test_units/random-bots-for-test.list b/dev-tools/test_units/random-bots-for-test.list index 96c9e99c2..ba412f871 100644 --- a/dev-tools/test_units/random-bots-for-test.list +++ b/dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -01h4x.com -360Spider +404enemy 80legs ADmantX +AIBOT +ALittle Client Abonti Aboundex -AhrefsBot -AiHitBot -Alligator -AwarioRssBot +AfD-Verbotsverfahren +Aipbot +Alexibot +AllSubmitter +Anarchie +Aspiegel +Asterias +Attach +AwarioSmartBot BackStreet -BackWeb Backlink-Ceck -BacklinkCrawler +Badass +Bandit +Bigfoot +Black Hole +BlowFish Boardreader -BotALot -Brandprotect Buddy -BuiltBotTough -BuiltWith -BuzzSumo -CATExplorador -CensysInspect +Calculon CheTeam +CherryPicker +ChinaClaw Claritybot -Cogentbot -Copyscape +Cliqzbot +Cloud mapping +Cocolyzebot Cosmos -CrunchBot -DIIbot -Demon -Deusu -Devil -DigitalPebble -Dirbuster -Discobot +CrazyWebCrawler +Crescent +Curious +DSearch +DataCha0s +Digincore +Disco +Discoverybot Dispatch -DomCopBot -DomainCrawler -Dotbot +DomainStatsBot +Domains Project Download Wonder -EMail Wolf -Ebingbong -EroCrawler +Drip +ECCP/1.0 +EMail Siphon Exabot -Express WebPictures +ExtLinksBot ExtractorPro -Extreme Picture Finder -Ezooms -FHscan -FemtosearchBot -Fimap -FlashGet -Foobot +Flunky +Fuzz +Fyrebot +G-i-g-a-b-o-t GT::WWW -GetRight -Getintent -Gigabot -GoZilla -GrapeFX -HTMLparser -Haansoft -HaosouSpider -Harvest +GetWeb +Go!Zilla +Go-Ahead-Got-It +Grabber +GrapeshotCrawler +GridBot Havij -HybridBot +Heritrix IDBot Image Fetch -InfoNaviRobot +Image Sucker +Indy Library +InfoTekies +Intelliseek InterGET InternetSeer -Iria Iskanie IstellaBot JOC Web Spider Jbrofuzz JetCar -JustView -Jyxobot +Jorgee Kinza Kozmosbot +LNSpiderguy LWP::Simple -Lanshanbot +Larbin +Leap LeechFTP -LexiBot -LieBaoFast +LibWeb +Lightspeedsystems +LinkScan LinkWalker Linkdexbot -LinqiaMetadataDownloaderBot -LinqiaScrapeBot -Lmspider -MFC_Tear_Sample +LinkpadBot +LinqiaRSSBot +Ltx71 +MIDown tool MQQBrowser -MSFrontPage -MSIECrawler +MTRobot Mag-Net Majestic-SEO +Mass Downloader Masscan -Mata Hari Mb2345Browser -MeanPath Bot +Meanpathbot +Mediatoolkitbot +MegaIndex.ru Metauri -Mojeek -Mozlila +MicroMessenger +Moblie Safari +Morfeus Fucking Scanner Mr.4x3 -Msrabot -Musobot -Name Intelligence +NPbot Nameprotect -Needle -NetAnts -NetSpider -NetZIP +Nessus +NetMechanic Netcraft -Netvibes +Nibbler Niki-bot NimbleCrawler -Nimbostratus +Nmap +Nutch Octopus Offline Explorer Offline Navigator -OnCrawl -OpenVAS -OrangeBot -OutclicksBot +OpenLinkProfiler +Openfind +OrangeSpider +OutfoxBot +PECL::HTTP +PHPCrawl +POE-Component-Client-HTTP +Page Analyzer +PageAnalyzer PageGrabber -PageScorer -Pavuk -Pi-Monster -Pixray -Pockey +Pandalytics +Panscient +Petalbot +Picscout +Picsearch +Pimonster ProPowerBot -ProWebWalker Psbot -Pump +PxBroker +PyCurl QueryN Metasearch -RSSingBot -RankFlex -RankingBot -RankingBot2 +RankActive +Rankivabot Re-re -RealDownload -Reaper -Recorder -RepoMonkey -RocketCrawler -SBIder +RedesScrapy SEOkicks -SISTRIX -SalesIntelligent +SMTBot +ScanAlert Scanbot +ScoutJet Scrapy +ScreenerBot +ScrepyBot SearchmetricsBot +Semrush SeoSiteCheckup SeobilityBot -Shodan SiteLockSpider -SlySearch -SmartDownload -Snoopy -SocialRankIOBot -Sosospider -Sottopop +SiteSnagger +Snake +Snapbot +Sociscraper +Sogou web spider +SpaceBison Spanner -Spbot -Spinn3r -SputnikBot -Sqlmap -Sqlworm -Steeler -Stripper +Sucker Sucuri SuperBot SuperHTTP -Surfbot SurveyBot -Suzuran Szukacz T0PHackTeam -T8Abot -TeleportPro Telesoft Telesphoreo -Telesphorep The Intraformant -Thumbor TightTwatBot +Toata Tracemyfile +Trendiction +Trendictionbot +True_Robot Turingos -TwengaBot -UnisterBot +Turnitin +URLy Warning +URLy.Warning V-BOT -VB Project VCI -VeriCiteCrawler +Vacuum Virusdie -Voltron +VoidEYE +WEBDAV WISENutbot -WWW-Collector-E WWWOFFLE -Wallpapers +Wallpapers/3.0 +WeSEE +Web Auto +Web Collage +Web Enhancer Web Fetch -Web Pix +Web Sauger Web Sucker -WebCollage -WebFuck -WebPix -WebReaper -WebSucker -WebWhacker +WebImageCollector +WebSauger +WebStripper +WebZIP +Webalta +WebmasterWorldForumBot +Webshag Website Quester -Whatweb +Whack Who.is Bot +WiseGuys Robot +Wonderbot Woobot -Zermelo -Zitebot +Wotbox +Xaldon WebSpider +Xenu +Zauba +Zeus ZmEu +ZoomBot ZumBot -ZyBorg -archive.org_bot arquivo-web-crawler arquivo.pt +autoemailspider +cah.io.community cognitiveseo -com.plumanalytics +crawl.sogou.com crawler4j -facebookscraper -gopher +evc-batch instabid -lwp-request -lwp-trivial +internetVista monitor +ips-agent +isitwp.com magpie-crawler -meanpathbot -muhstik-scan -polaris version -probe-image-size -scalaj-http +page scorer +pcBrowser +s1z.ru scan.lol -seobility seoscanners -siteripz -sogouspider -spyfu -tAkeOut -trendiction.de -voyagerx.com +seostar +serpstatbot +sysscan +trendiction.com webmeup-crawler x09Mozilla -x22Mozilla xpymep1.exe -zauba.io diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 37230dff8..03540b6b7 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -404checker -AIBOT -ALittle Client +01h4x.com +80legs ASPSeek Abonti -Acunetix -AfD-Verbotsverfahren -AiHitBot -Anarchie +Aboundex +Anarchy99 +Anthill Aspiegel -Asterias -AwarioSmartBot +Attach BBBike -BDCbot -BDFetch -BackDoorBot +BackStreet +Backlink-Ceck BacklinkCrawler -Barkrowler +Bandit BatchFTP -Bitacle +Black Hole +BlackWidow Blackboard -BotALot +Blow +BlowFish Brandprotect -Buddy +Buck +BuiltWith +Bullseye +CATExplorador CCBot +CODE87 CSHttp -Calculon -CazoodleBot Cegbfeieh -CensysInspect CheTeam -CheeseBot +CherryPicker +Chlooe Claritybot +Cliqzbot Cloud mapping -Cogentbot -Copier +Cocolyzebot +Collector CopyRightCheck -Cosmos -Craftbot -CrunchBot -Custo -DBLBot -DatabaseDriverMysqli +Curious +CyotekWebCopy +DIIbot +DTS Agent Demon -Devil Digincore +DigitalPebble +Disco +Discobot Discoverybot -DomCopBot -DomainAppender +DnyzBot DomainCrawler DomainSigmaCrawler -DomainStatsBot Domains Project -ECCP/1.0 -EMail Siphon -EirGrabber -Exabot +Dotbot +EMail Wolf +Ebingbong +EroCrawler Express WebPictures +ExtLinksBot Extractor -Ezooms -FDM +ExtractorPro +Extreme Picture Finder +FHscan FemtosearchBot +Fimap +Firefox/7.0 FlashGet -Foobot +Flunky Freeuploader FyberSpider G-i-g-a-b-o-t -GT::WWW GermCrawler -GetRight -GetWeb -Getintent -Gigablast -GoZilla +Gigabot +Go-Ahead-Got-It Gotit -GrabNet -Haansoft +Grafula +GrapeFX +GridBot +HTTP::Lite HaosouSpider -Harvest Havij -Hloader -IRLbot -Image Fetch -IndeedBot -Indy Library -InfoNaviRobot -Intelliseek -InternetSeer -Iria -IstellaBot +HybridBot +IDBot +Iblog +IlseBot +Image Sucker +InfoTekies +Iskanie +Jbrofuzz JetCar Jetty JikeSpider Joomla -Jorgee -Leap -LeechFTP -LexiBot -Linkbot -LinkextractorPro -LinkpadBot -LinksManager -LinqiaRSSBot +Jyxobot +Keyword Density +LNSpiderguy +Larbin +LeechGet +LibWeb +Libwhisker +LinkScan +LinkWalker +Linkdexbot Lipperhey +Lipperhey Spider Litemage_walker -MFC_Tear_Sample MIIxpc MQQBrowser -Mag-Net +MSFrontPage +MSIECrawler Magnet -Majestic SEO -Majestic12 Mass Downloader -MeanPath Bot -Metauri -MicroMessenger +MauiBot +Mb2345Browser +Meanpathbot Microsoft Data Access Mister PiX Moblie Safari -Mojolicious -Mr.4x3 -Musobot -NICErsPRO -Nameprotect -Nessus -NetMechanic +Msrabot +NearSite +Net Vampire +NetLyzer Netcraft -Nibbler -Niki-bot -NimbleCrawler +NextGenSearchBot Nimbostratus -Ninja Nutch -Octopus Offline Explorer OrangeBot OutclicksBot -PECL::HTTP +OutfoxBot +POE-Component-Client-HTTP +Page Analyzer PageAnalyzer PageGrabber PageThing.com +Pandalytics +Panscient Pavuk Petalbot +Pi-Monster +Picsearch Piepmatz -Pixray +Pimonster +PleaseCrawl +ProPowerBot +Probethenet Psbot Pu_iN -PxBroker -RankActive +Quick-Crawler RankActiveLinkBot -RankurBot +RankFlex +RankingBot2 +Rankivabot ReGet -RebelMouse -Recorder +RealDownload RedesScrapy RocketCrawler -SEOstats -SalesIntelligent +SBIder +SEOkicks +SEOprofiler +SISTRIX +ScanAlert Screaming +ScreenerBot ScrepyBot -SearchmetricsBot -SeoSiteCheckup -Shodan +SeobilityBot Site Sucker SiteCheckerBotCrawler +SiteExplorer +SiteLockSpider SiteSnagger -Sitebeam -Siteimprove Sitevigil -SmartDownload +SlySearch +Snake Snapbot -Spbot +SpankBot Spinn3r SputnikBot -Sqlmap -Stripper -SuperHTTP +Sqlworm +Sucker +Sucuri +SuperBot +Surfbot SurveyBot -Suzuran Swiftbot -Szukacz -T0PHackTeam +T8Abot Teleport -Telesphoreo +The Intraformant +Thumbor TightTwatBot Toata -True_Robot +Toweyabot +Tracemyfile +Trendiction Turingos TurnitinBot -Twice -URLy Warning +TwengaBot +Typhoeus UnisterBot V-BOT -VB Project +VCI Vacuum -VelenPublicWebCrawler -VidibleScraper +VeriCiteCrawler Virusdie -VoidEYE -Voil -WASALive-Bot WEBDAV -WISENutbot -WallpapersHD -Web Auto -Web Collage -Web Fuck -Web Pix -Web Sucker +WWW::Mechanize +Wallpapers +Web Enhancer +WebAuto WebBandit WebCollage -WebGo IS +WebCopier +WebEnhancer +WebFetch WebLeacher -WebPix -Webalta -Website Quester -Whack +WebsiteExtractor +WebsiteQuester Whacker Whatweb Who.is Bot -WinHTTrack -Xaldon WebSpider -YoudaoBot -Zermelo -Zeus -ZoomBot -ZumBot -archive.org_bot +Widow +Wotbox +Zade +Zauba +ZmEu +arquivo.pt backlink-check +cah.io.community check1.exe +cognitiveseo +com.plumanalytics crawler4j -demandbase-bot -facebookscraper +gopher heritrix +instabid +internetVista monitor +ips-agent isitwp.com -magpie-crawler -mediawords -netEstate NE Crawler +muhstik-scan +page scorer pcBrowser -probe-image-size +polaris version s1z.ru -satoristudio.net +scalaj-http +scan.lol +seobility seoscanners -serpstatbot -sogouspider -sp_auditbot -trendiction.com -webmeup-crawler -xpymep1.exe -zauba.io +sitechecker.pro +sysscan +voyagerx.com +zgrab diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index fcf98c6ef..db22bbdf0 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -energydiet-info.ru -everypony.ru -ezofest.sk -gelezki.com -kiinomaniak.pl -megakino.net -officedocuments.net -rockma.se -space-worry.ml -vertaform.com +cornomase.win +erotik-kostenlos.net +investzalog.ru +jerseysbizwholesalecheap.com +medtherapy.ru +poiskzakona.ru +speedup-my.site +trumpetedextremes.com +umekana.ru +videosbox.ru diff --git a/dev-tools/test_units/random-referrers-for-test.list b/dev-tools/test_units/random-referrers-for-test.list index 120bc25ce..950c50541 100644 --- a/dev-tools/test_units/random-referrers-for-test.list +++ b/dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -0lovespells0.blogspot.com 1001watch.com.ua -100searchengines.com -101billion.com +18ps.ru 1hwy.com -2020iscoming.info -202ch.com -3w24.com -4kplayer.pl -883zy.com -8lufu.com -9icmzvn6.website -a.pr-cy.ru -accessoires-mode.in -adsfresh.men -adultnet.in -advancedmassagebysara.com -advokateg.ru +2daytrendingnews.com +4kepics.com +4pp13.com +5000-cotydzien.com +5i2.net +70casino.online +8xv8.com +aa9bd78f328a6a41279d0fad0a88df1901.com +aaa38852e886ac4af1a3cff9b47cab6272.com +aaronabel.com +absugars.com +addbin.men +addblueoff.com.ua +admitad.com +ads-cool.pro +ads-seo.men +adultfriendfinder.com +adultmeetonline.info afmuseum.com -air-edem.ru -alert.scansafe.net -alienwheels.de -allesohnegirls.net -allsilver925.co.il +ageofclones.com +ahmedabadwebs.com +akusherok.ru +alekseevec.ru +alert-fdm.xyz +allcredits.su +allnews.md +allpdfmags.net allstatesugarbowl.org -amazingpic.net -ananumous.ru +amateurmatch.com anaseracresar.tk -android-systems.ru -android-vsem.org +angel1777.kz angkortours.vn -animal-drawings.com -aosheng-tech.com -apartmentratings.com -apidigidocketnet-a.akamaihd.net -apiportalmorecom-a.akamaihd.net -appearance-cool.com -artavenuegardenstudios.com +angry-fermi-7633.arukascloud.io +ap.senai.br +apifasterlightin-a.akamaihd.net +apparel-offer.com +appsecurityr.com +arabsexxxtube.com +arclk.net +arvut.org +asiavirtualsolutions.com asociatia-tipografilor-transilvania.ro -asrv-a.akamoihd.net -avon-ukraine.com -ayakino.net -azartclub.org -babespcs.com -badavit.com.ua -baikaleminer.com -bambi.ck.ua -bang-hotties.com -beremenyashka.com -best-businessman.ru +asscenihotosidea.blogspot.co.za +australia-opening-times.com +autogrep.ru +autotrends.today +autovideobroadcast.com +avitocars.ru +avto-oligarh.ru +ayanaspa.com +backlink4u.net +basedecor.ru +battlecarnival.su +belsetirehafi.tk +best-deals-products.com best-mam.ru +best-seo-solution.com besthoro.ru -bestmarriages.com -betterscooter.com -bigcareer.info -bio-japan.net -bioinnovate.co -boo-arts.com -boole.onlinewebshop.net -bouda.kvalitne.cz -brainboosting.club -braintobe.win -brakehawk.com -buigas.00it.com -business-made-fun.com -buttons-for-website.com -buttons-for-your-website.com -buy-cheap-pills-order-online.com -cakesplus.com.au -callejondelpozo.es -canrole.com +bestserials.com +bestwaystogetpaid.us +bestwebsiteawards.com +betteroffers.review +bfz.biz +bidbuy.co.kr +biglistofwebsites.com +billiard-classic.com.ua +bioca.org +biocn.dx.am +bioscorp.ru +bizcheapjerseyswholesalechina.com +bjanshee.ru +blackle.com +blagovest-med.ru +blobar.org +blockworld.ru +blogsfunky672.weebly.com +bluesalt.co +bluesman.nu +bonux.nextview.ru +brg8.com +btvn.ru +bukleteg.xyz +buyessayonline19.blogspot.ru +buzz-porno.info +callmd5map.com +canoncdriverq3.pen.io canvas.gloverid.site -carapics.com -cdncash.org -cheatcode-lita12.rhcloud.com -chlooe.com -christianlouboutinoutlet.win -clean-virus-mac.com -colobit.biz -comeondog.info -compliance-mark.top -contentlook.co -cool-wedding.net -copenergo.ru -coprofam.org -cosmetic.donna7753191.ru -cowlmash.com -craftburg.ru -creatives.adbetclickin.pink -crynet.cc -cscwtalkto.site -customcollegeessays.net -cv.wallhade.co -darkages.info -db.speedup.gdn -decorationspcs.com -deniven.1bb.ru -depositfiles-porn.ga -descargar-musica-gratis.net -deutschehobbyhuren.net -dev-seo.blog -dikqlyremy.info -dispo.de -djekxa.ru -dlya-android.org -dnepr-avtospar.com.ua -dnsrsearch.com -docs4all.com -docstoc.com -dorratex.tn -downloads-whatsapp.com -downtuptv.gq -dstroy.su -dumpsmania24.com -dyshagi.ru -earncash.com.ua -easycommerce.cf -ecommerce-seo.com -editmedios.com -elektir.ru -elektronischezigarettekaufen1.myblog.de -eloconcream.blogspot.com -elvenar.com -enskedesquashclub.se -enthuse.computernetworksonline.com -eredijovon.com -erotik-kostenlos.net -este-line.com.ua -eu-cookie-law.blogspot.com -eupornstar.info -europeanwatches.ru -everytalk.tv -exportshop.us -expresstoplivo.ru -extlabs.io -extremez.net +cartierbracelet.xyz +ccjp.eu +cenoval.ru +centrdebut.ru +cheapsergiorossi.online +chocolateslim-en-espana.com +christianlouboutinreplica.pw +clarithromycin500mg.com +clickbanksites.info +clickprotects.com +cognitiveseo.com +collegeessay19.blogspot.ru +com-onlinesupport.host +cometorussia.net +communisave.co.za +compliance-brian.top +compliance-julia.top +consorzioilmosaico.org +cougfan.info +counterbot.com +country-chic.ru +covid-schutzmasken.de +cpajunkies.com +crackguru.tk +creams.makeforum.eu +cupday.com +datract.com +de.zapmeta.com +deadlinkchecker.com +delta-line.men +demian.kz +detsky-nabytek.info +devochki.top +dinosaurus.site +disruptingdinnerparties.com +divatraffic.com +doctissimo.fr +dogclothing.store +dogsrun.net +domain.webkeyit.com +domznaniy.ru +doska-vsem.ru +dotnetdotcom.org +doxysexy.com +drpornogratisx.xxx +druzhbany.ru +ebonyporn.site +ecookna.com.ua +edelstahlschornstein-123.de +efkt.jp +ekspertmed.com +engines-usa.com +eorogo.top +erotag.com +es5.com +eshop.md +espaceinventoristes.com +etur.ru +eu-cookie-law.info +ewebarticle.info +exoneration-project.us +expolicenciaslatam.co eyessurgery.ru -ezigarettekaufen2.mpbloggar.se ezigarettenshop1.yolasite.com -ezigaretteshop2.mywapblog.com -f-online.de -favorcosmetics.com -fetishinside.com -filesdatabase.net -filunika.com.ru -findpik.com -fingerprintjs.com -firesub.pl -forexunion.net -formularz-konkurs.tk -forodvd.com -foto-telok.net -fotopop.club -fotosfotos.eu -free-share-buttons.blogspot.com -freelotto.com -freemags.cc -frustrated-favorable.gq -frvo.alptandem.ru -fuckbuddybestgilf.info -gamedayhouse.com -gavuer.ru -gay.adultgalls.com -gdebestkupit.ru -gearsadspromo.club -generalporn.org -getlamborghini.ga -gidonline.one -gilbertbanda.net +f1nder.org +fanpagerobot.com +faptitans.com +fashionavenuegame.com +fba-mexico.com +feorina.ru +ferieboligkbh.dk +fickenbumsen.net +fickluder69.com +filmci.pro +finejewelryshop.ru +flauntyoursite.com +folowsite.com +formulaf1results.blogspot.com +forum.doctissimo.fr +foto-weinberger.at +foxtechfpv.com +fres-news.com +freshberry.com.ua +froggytube.com +fuck-paid-share-buttons.xyz +fuckingawesome.com +fungirlsgames.net +g.starmoe.xyz +galaxyflowers.ru +gamerscorps.com +ge0ip.com +getprismatic.com +gheus.altervista.org +girls-ufa.ru glicol.kz -gogps.me -gok-kasten.net -golfresa.lucania.se -goodhousekeeping.com -googlsucks.com -gorodservis.ru -gosmeb.ru -gotcher.us -gotomontenegro.net -gymi.name -hayate.biz -hdhc.ru -headpharmacy.com -healgastro.com -health-medical-portal.info -hebr.myddns-flir.com -high-speed1.net -hikesearch.net -hkdiiohi.skyrock.com -homedo.fabpage.com -homemade.gq -honyaku.yahoofs.jp -hostcritique.com -hundejo.com -hydropump.su -ico.re +globatur.ru +goblacked.com +goldenggames.com +gombita.info +gomusix.com +gonextmedia.com +goodnightjournal.com +googglet.com +grand-chlen.ru +gratuitbaise.com +gratuitxblcodes.com +gsasearchenginerankerhelp.com +guidefs.ru +hauleddes.com +hchha.com +hdfreeporno.net +healbio.ru +healmytrauma.info +hem.passagen.se +highland-homes.com +homeinns.com +hoste.octopis.com +hostingclub.lk +hotblog.top +hotelcrocenzi.sm +huimin764128.com +humanelydrew.com +hybrid.ru +idealtits.net +ie.57883.net +iflycapetown.co.za +iframe-toloka.com igtools.club -images-free.net -imgata.com -iminent.com -imk.com.ua -infobanks.ru -inmate-locator.us -it-max.com.ua -itronics.ca -izhevsk.zrus.org -jagg.info -javidol.site -jerseychinabizwholesale.us -kamen-e.ru -kazka.ru -kicknights.gq -kinoduh.ru -kinostorm.net -kiwi237au.tk -knogg.net -konteiner24.com -kumuk.info -kutikomi.net -lacloop.info -lanasshop.ru -landoftracking.com -lankarns.com -lapitec.eu -larchik.net -leadwayau.com -leftofcentrist.com -li-er.ru -lifehacĸer.com -liky.co.ua -lion.bolegapakistan.com -lion.conboy.us -loginduepunti.it -lost-alpha.ru -lutherstable.org -m4ever.net -machicon-akihabara.info -magnetic-bracelets.ru -mainlinehobby.net +iklysha.ml +images-graphics-pics.com +immobiliaremassaro.com +intim-uslugi.info +irunfar.com +iscblog.info +itis4you.com +iwanttodeliver.com +izoll.ru +jacago.com +jetsli.de +johannesburgsingles.co.za +joy-penguin.com +justkillingti.me +jwss.cc +kaliningrad.zrus.org +kandidos.com +kinohit1.ru +kinosed.net +krassh.ru +kreativperlen.ch +kuzinsp.ru +kw21.org +labvis.host.sk +ladsup.com +larutti.ru +law-check-nine.xyz +lazy-z.com +ldrtrack.com +le-clos-des-alouettes.com +lemon-ade.site +lesbian.xyz +light.ifmo.ru +likesuccess.com +likrot.com +linkbolic.com +liquimondo.com +listiseltemournlan.gq +livefixer.com +liver-chintai.org +livesex-amateure.info +liveu.infoteka.hu +lmrauction.com +loftdigital.eu +lomb.co +lovelycraftyhome.com +lulea-auktionsverk.se +luxmagazine.cf +m-google.xyz +m1media.net +mackeeper-land-672695126.us-east-1.elb.amazonaws.com +mail.allnews24.in +makedo.ru +makemoneyonline.com makenahartlin.com -malwareremovalcenter.com -manifestyourmillion.com -marketingvici.com -maxxtor.eu -mcnamaratech.com +mall.uk +marmotstore.online +maximilitary.ru +medicovi.com medispainstitute.com.au -medizinreisen.de -mensandals.xyz -migente.com -mikrobiologies.ru -milblueprint.com -mini.7zap.com -mir-business-24.ru -misswell.net -mixx.com +member-quiz.com +mesbuta.info +mesto-x.com +metascephe.com +mex-annushka.ru +midst.eu +militarysale.pro +miniature.io +mlf.hordo.win mnogabukaff.net -mobplayer.ru -monclerjacketsoutlet.win -moneyzzz.ru -morlat.altervista.org -moviezbonkerssk.cf -mp3ringtone.info -mrbojikobi4.biz -mriyadh.com -mulberryoutletonlineeu.com -music7s.me -muz-shoes.ru -muzaporn.com -myblogregistercm.tk -mymobilemoneypages.com -nacap.ru -naj-filmy24.pl -napalm51.nut.cc +mobplayer.net +moesen-ficken.com +moneyteam24.com +montazhnic.ru +morocco-nomad-excursions.com +moroccosurfadventures.com +mp3s.club +musezone.ru +musicas.baixar-musicas-gratis.com +mycouponizemac.com +myfreecams.com +myprintscreen.com +na15.ru +nagdak.ru +naperehresti.info +nastroyke.net +natali-forex.com nbsproject.ru -new-post.tk -nicovideo.jp -njkmznnb.ru -nl.netlog.com -nodding-passion.tk -nofreezingmac.work -noumeda.com -novodigs.com -nudepatch.net -odesproperty.com -offer.camp -office-windows.ru -offside2.5v.pl -oivcvx.website +net-profits.xyz +notaria-desalas.com +npoet.ru +nrjmobile.fr +nw-servis.ru +oakleyglassesonline.us +oballergiya.ru +oda.as +offf.info +okmedia.sk +oktube.ru olvanto.ru -one.net.in -online-hit.info -ontargetseo.us -ooo-gotovie.ru -orgasmatrix.com -pateaswing.com -pay2me.pl -paydayloanslocal.com -pcimforum.com +online-zaymy.ru +onlinemeetingnow.com +onlythegames.com +openfrost.net +openstat.com +outshop.ru +paceform.com +panouri-solare-acoperis.com +paperwritingservice17.blogspot.ru +parajumpersjakkesalgnorge.info +paramountmarble.co.uk +partnersafe.men +pathwhelp.org +pc-virus-d0l92j2.pw +pdns.cz perfectpracticeweb.com -perm.xrus.org -photo-clip.ru -piccshare.com -pinapchik.com -piratecams.com -pizdeishn.com -pizza-imperia.com -playlott.com -porno-chaman.info -pornobrazzers.biz -pornodojd.ru -pornofiljmi.com -pornoslive.net +pestomou.info +petedrummond.com +php-market.ru +pierrehardysale.online +piluli.info +pixgood.com +pl.aasoldes.fr pourvous.info -pozdravleniya-c.ru -pr0fit-b0x.com +pr-ten.de praisong.net -predmety.in.ua -pricheskaonline.ru +pravoholding.ru +preparevideosafesystem4unow.site +privacyassistant.net privacylocationforloc.com -privatov-zapisi.ru -privetsochi.ru -projectforte.ru -projefrio.com.br -prombudpostach.com.ua -pron.pro -prostitutki-tyumeni.org -prpops.com -push-ad.com -questionmarque.ch -quickchange.cc -quickloanbank.com -r-e-f-e-r-e-r.com -rainbowice.ru -razleton.com -reelheroes.net -reeyanaturopathy.com -regdefense.com -reklama1.ru -remont-ustanovka-tehniki.ru -remontvsamare.su -renecaovillasale.online -rfserial.net -riders.ro -rukino.org -ruscoininvest.company -rusoft-zone.ru -sabizonline.com -sampleletters.net -savingsslider-a.akamaihd.net -sawin.beth.webd.pl -scanner-andrew.top -scanner-jess.top -school-diplomat.ru +prizeestates.cricket +pro-poly.ru +proekt-gaz.ru +proekt-mos.ru +profolan.pl +prohoster.info +pssucai.info +pushdata.sendpulse.com +putitin.me +pyrodesigns.com.au +qor360.com +ranksonic.com +recipedays.com +registry-clean-up.net +ric.info +riralmolamsaca.tk +rniaeba.ga +rocketchange.ru +rollercoin.com +romhacking.ru +royal-investments.net +ruscopybook.com +russintv.fr +russkie-gorki.ru +rustic-quiver.win +sabaapress.com +saratov.xrus.org +scanner-alex.top +schoolfiles.net +scopich.com score-ads.men -screentoolkit.com -sdi-pme.com -sdsjweb.com -search-goo.com -searchmywindow-a.akamaihd.net -secret.xn--oogle-wmc.com -security60-e.com -segol.tv +searchtooknow-a.akamaihd.net +seasaltwithfood.com +seasonvar.ru +sei80.com seinterface.com -seo18.su +seo-tools-optimizing.com +seocheki.net seogadget.ru -seorank.info -serdcenebolit.com -servisural.ru -sex-sex-sex5.com -sexobzor.info -sexreliz.com -sexs-foto.com -sextracker.de -sexy.babes.frontend-stack.top -shiksabd.com -shopsellcardsdumps.com -shtaketniki.kz -sideeffectsoftizanidine.blogspot.com +seojokes.net +seokicks.de +sexuria.net +sexystrippe.info +shisha-swag.de +shitting.pro +shoppingmiracles.co.uk +signoredom.com +signx.info +similardeals.net site-auditor.online -small-game.com -smichovbike.cz -soc-econom-problems.ru -social-fun.ru -social-search.me -socialmediasuggest.com -softtor.com -softwaretrend.net -sohoindia.net -sophang8.com -soserfis.com -soundfrost.org -spbchampionat.ru -specialfinanceoffers.com -speechfoodie.com -ssconstruction.co -star61.de -steelmaster.lv -stockquotes.wooeb.com -strag-invest.ru +siteaero.com +siteimprove.com +sixcooler.de +snegozaderzhatel.ru +social-buttons.com +sockshare.net +sofit-dmd.ru +soheavyblog.com +spammen.de +spamnuker.com +steebook.com +stickers-market.ru +strana-solnca.ru +streetfooduncovered.com +strfls.com +stroilka.info suggest-keywords.com -supernew.org -supers.com.ua -svetlotorg.ru -takeprofitsystem.com -tampabaywatch.org -taranerymagesswa.blogspot.com -teenbbw.yopoint.in -templates.franklinfire.co -thecounter.com -thedownloadfreeonlinegames.blogspot.com -thetardistimes.ovh +superlist.biz +surcentro.com +surflinksmedical.com +surgut.zrus.org +svarbit.com +symphonyintegratedhealthcare.com +taaaak.com +taraz.xkaz.org +tattomedia.com +tattooreligion.ru +tds-advert005.info +teastory.co +technika-remont.ru +telegraf.by +telsis.com +tengohydar.tk +terraclicks.com +tetracsaudi.com +texbaza.by +thelottosecrets.com tiens2010.ru -tizanidinecost.blogspot.com -tizanidinemusclerelaxant.blogspot.com -tizanidinepills.blogspot.com -tizanidinesideeffects.blogspot.com -todohr.com -tootoo.to -top-karkas.ru -topkarkas.com -torrent-to-magnet.com -torrents-tracker.com -torrnada.ru -toursmaps.com -track.deriv.com -trainoffend.ml -trubywriting.com -trucri.me -trustedmaccleaner.com -tryrating.com -tvnewsclips.info -uhod-za-sobakoj.ru -ukr-lit.com -uptime-as.net -upupa.net -urlopener.com -userequip.com -vacuumcleanerguru.com -vashsvet.com -vchulkax.com -veopornogratis.xxx -viagengrarx.com -viberdownload10.com -videochat.mx -videositename.com -videtubs.pl -vigrx-original.ru -vinsit.ru -vipsiterip.org -visitcambridge.org -viven.host.sk -vkmusics.ru -vladhistory.com +timetorelax.biz +tizanidineingredients.blogspot.com +tizanidinewithdrawalsymptoms.blogspot.com +tonerbox.kz +torture.ml +trafficfactory.biz +trahic.ru +trkur.com +truck-land.life +tuberkulezanet.ru +twiclub.in +twitlinks.com +valuado.com +venta-prom.ru +video--production.com +videochat.world +videotuber.ru +virus-respirators.com +visa-china.ru vseuznaem.com -vzubah.com -walpaperlist.com -wardreapptokone.tk -webinstantservice.com -weblibrary.win +vykupavto-krasnodar.ru +vzlom-na-zakaz.com +vzlomtw.com +w7s.ru +walkme.com +watchinf.com +web-analytics.date +webadvance.club +webix.me +webjam.com website-analytics.online -website-datenbank.de -website-speed-check.site -websiteaccountant.de -webtherapy.ru -wellcome2slovenia.ru -whatzmyip.net -wildnatureimages.com -winx-play.ru -wjgony.com -wnoz.de -woman-orgasm.ru -woodyguthrie.se -wordpresscore.com +whengirlsgowild.com +wonderfulflowers.biz worldis.me -worst-sites.online -writingservices17.blogspot.ru -www.event-tracking.com -www.wohnkabinen-shop.de -x-diesel.biz -x5market.ru -xboxster.ru -xcc24.pl -xgames-04.com -xn----7sbahjd3btneuw1joc.xn--p1ai -xn--80abgj3a5acid6ghs.top +wrona.it +wtsindia.in +xclicks.net +xerox-douglas.cf +xingzi-vision.com +xitjw.info +xn----8sbhefaln6acifdaon5c6f4axh.xn--p1ai +xn----itbeirbjbi7bc6bh2d.xn--p1ai +xn--80ahvj9e.xn--p1ai xn--b1agm2d.net -xxlargepop.com -xxxmania.top -youdesigner.kz -youporn-forum.ga -yourothersite.com +xtraffic.plus +xxxnatelefon.ru +youbloodyripper.com +youradulthosting.com +yousense.info yun56.co +zaapplesales.blogspot.com zajm-zalog-krasnodar.ru -zebradudka.com -zeroredirect8.com -zigarettenonl.canalblog.com -zootoplist.com -zootravel.com -zrus.org -zs2vm.top +zaobao.com.sg +zarajbuilders.com +zarepta.com +zed21.net +zeikopay.com +zeleznobeton.ru +zeroredirect5.com +zeroredirect9.com +znakomstvaonlain.ru +zreloeporno.tv +zvetki.ru +zzlgxh.com diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index a5ddfdd73..642a2b6ce 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -007angels.com -3w24.com -4pp13.com -5000-cotydzien.com -51unlim.ru -5tey7463.weebly.com -66cpwgln.space -72-news.com -aac500b7a15b2646968f6bd8c6305869d7.com -advocatemsk.ru -advokateg.xyz -aibolita.com -arquapetrarca.info -autovideobroadcast.com -avkzarabotok.info -azbuka-mo.ru -bashtime.ru -beauby.ru -belinka.com.ua -best-seo-software.xyz -bestwebsitesawards.com -betune.onlinewebshop.net -bif-ru.info -bigames.online -bitcoins-live.ru -bloggedporn.com -bookhome.info -brandehk.dk -brendbutik.ru -brokergid.ru -brothers-smaller.ru -buy-meds24.com -candcstuccoandstone.com -capsderfudd.tk -cartechnic.ru -cdncash.org -cejewelry.xyz -cete.ru -cfjump.com -cheapcigarettesc.info -checkm8.com -collegerentals.com -conciergegroup.org -coolgramgoods.com -crirussian.ru -darodar.com -dating-app.ru -davebestdeals.com -diegolopezcastan.com -digital-sale.su -dignitasdata.se -dispo.de -dmmspy.com -dms-sw.ru -dood.live -doyouknowtheword-flummox.ml -drugstoreforyou.com -easyukraine.com -ecommerce-seo.org -ege-russian.ru -elektrischezigarettekaufen2.cowblog.fr -elite-sex-finders.com -esopini.com -everytalk.tv +00author.com +03p.info +18ps.ru +24x7-server-support.site +4senses.co +5-steps-to-start-business.com +aarbur.com +acgs.tk +addbin.men +adhome.biz +adserver-e7.com +adultfullhd.com +afora.ru +agadelha.com.br +akuhni.by +alborzan.com +alert-fjg.xyz +alice.it +alldezservice.kz +allwidewallpapers.com +android-style.com +arcadeplayhouse.com +arclk.net +atelielembrancaqueencanta.com.br +autorn.ru +b00kmarks.com +banki76.ru +bausparen.bz.it +bdsmgalls.net +beachdriveblog.com +beauty-clinic.ru +bimatoprost-careprost.com.ua +bioinnovate.co +biznesluxe.ru +brainboostingsupplements.org +brains2.biz +buckspar.gq +business-made-fun.com +camdenmemorials.com +canyougethighofftizanidine.blogspot.com +cartierjusteunclou.xyz +check-this-out-now.online +chocolateslim-u-srbiji.com +ciceros.co +classicakuhni.ru +compliance-andrew.top +compliance-checker-7.info +compliance-fred.top +compliance-jessica.top +contextualyield.com +corta.co +cowblog.fr +craftinsta.ru +creams.makeforum.eu +cymbaltaandtizanidine.blogspot.com +cyse.tk +d2jsp.org +darkages.info +deadlinkchecker.com +demian.kz +dentalpearls.com.au +deshevo-nedorogo.ru +dikqlyremy.info +dinosaurus.site +documentsite.net +domination.ml +doreenblog.online +download-of-the-warez.blogspot.com +doxysexy.com +drhomes.biz +drugs-no-rx.info +educontest.net +elektronischezigarettekaufen2.tumblr.com +elektrozigarette1.dreamwidth.org +elmifarhangi.com +espaceinventoristes.com +evaashop.ru extlinks.com -factorynightclub.com -faithe.top fashion-mk.net -ferrotodo.com -forexgb.ru -foxydeal.com -freenom.link -freshberry.com.ua -g33.org -galeria-zdjec.com -gameprimary.com -gne8.com -gojiberriess.apishops.ru -google-liar.ru +faz99.com +fertilitetsradgivningen.se +filerockstar298.weebly.com +filesclub.net +find1friend.com +finstroy.net +fishingwholesale.us +forminecrafters.ru +formulaantiuban.com +foto-telok.net +froggytube.com +fullfileaccess.com +fyl.com.ru +gay.adultgalls.com +ghazel.ru +ghernnqr.skyrock.com +gomusix.com +googlsucks.com +greenidesign.co growmyfunds.ca -gwebtools.com.br -helvetia.com.ua -hentaiheroes.com -highspeed5.net -horoshieokna.com -hoste.octopis.com -hotcore.info -hptwaakw.blog.fc2.com -hundejo.com -hvd-store.com -hyiphunter.org -i-hobot.ru -imadedinner.net -indianmedicaltourismshop.com -inspiring-desperate.tk -integritylandscapeservices.com -investmac.com -jerseychinabizwholesale.us -jimmychoosale.online -journeydownthescale.info -joy-penguin.com -kamalsinha.com -karpun-iris.ru -karusel-market.ru +gsmlab.pl +hamptonoaks.ca +historichometeam.com +hotblognetwork.com +hyhj.info +ido3.com +igru-xbox.net +ikritikimou.gr +in-tandem.co +kam-dom.ru +keywordhouse.com kiinomaniak.pl -klejonka.info -koptims.tiu.ru -kredit-blog.ru -laapp.com -labvis.host.sk -lafriore.ru -languagecode.com -larutti.ru -law-check-nine.xyz -lechenie-gemorroya.com -legionalpha.com -lenvred.org +kinobaks.com +la-fa.ru +lacave.ntic.fr +landreferat.ru +larchik.net +larose.jb2c.me +lawyersinfo.org +ledis.top lflash.ru -lion.bolegapakistan.com -loadopia.com -longgreen.info -louisvuittonoutletstore.net +linkarena.com +lumb.co +lunamedia.co m-google.xyz -malls.com -marcogrup.com -marketingtechniques.info -maxxtor.eu -meble-bogart.info +max-eclat.men +meddesk.ru megaapteka.ru -meltwater.com -menhealed.net -michaelkorsoutlet.store -misandesign.se -monthlywinners.com -mriyadh.com -mydearest.co -myplaycity.com -mysex21.com -nash-krym.info -nbsproject.ru -newstudio.tv -offer.camp -oktube.ru -onlainbesplatno.ru +metallosajding.ru +metarip.ru +mex-annushka.ru +michaelkorsoutletstore.net +miniads.ca +mitrasound.ru +mnimmigrantrights.net +montazhnic.ru +most.gov.iq +my-aladin.com +mydoctorok.ru +myfreemp3.eu +myonigroup.com +netallergy.ru +nordstar.pro +northfacestore.online +nowtorrents.com +nudejapan.net +nzfilecloud.weebly.com +oneshotdate.com +online-hit.info +online7777.com +onlineporno.site onlinewritingjobs17.blogspot.ru -onlyforemont.ru -ooo-gotovie.ru -ooo-olni.ru -outshop.ru -ownshop.cf -paceform.com -palocco.it -pdns.cz -perfectpracticeweb.com -phormchina.com -pixgood.com -pogruztehnik.ru -poisk-zakona.ru -pony-business.com -pornofeuer.com -pornoklad.net -portadd.men -pozvonim.com -preparevideosafesystem4unow.site -pro-poly.ru -profitsport.club -projefrio.com.br -prostitutki-almata.org -putitin.me -q-moto.ru -rangjued.com -ranksonic.com -razyboard.com -realitykings.com -redirect.trafficreceiver.club +ooomeru.ru +oynat.info +pacificair.com +partner-stop.men +picture-group.com +pictures-and-images.com +picturesfrom.com +piulatte.cz +piuminiita.com +pixelrz.com +podshipniki-ntn.ru +popugaychiki.com +porno-transsexuals.ru +pornogad.com +pornosee.info +pornowarp.info +pourvous.info +professionaldieselcare.com +profitkode.com +proxyradar.com +puteshestvennik.com +realting-moscow.ru +registry-clean-up.net +registry-cleaner.net +reining.lovasszovetseg.hu reward-survey.net +rfd-split.hr rockingclicks.com -rustic-quiver.win -saecsa.co -sandhillsonline.com -saneyes.com -sanyuprojects.com -savetubevideo.com -scansafe.net -sdelai-prosto.ru -seadragonherbery.com -seeresultshub-a.akamaihd.net -sendearnings.com -sexsaoy.com -sierraapps.com -skachat-besplatno-obrazcy.ru -slujbauborki.ru -smartshoppymac.com -snsdeainavi.info +rostov.xrus.org +royalads.net +royalcar-ufa.ru +ruclicks.com +santechnik.jimdo.com +sax-sex.com +scanner-jack.top +scanner-margo.top +scat.porn +search-goo.com +securesmrt-dt.com +serdcenebolit.com +serptehnika.ru +service.adtech.fr +sideeffectsoftizanidine.blogspot.com +simplesite.com +siteheart.net +siteripz.net +snjack.info soblaznu.net socas.pluto.ro -socialmonkee.com -sodexo.com -sohoindia.net -soundfrost.org -soviet-portal.do.am -spammen.de -spywarebegone.com -stackthatbucks.com -start.myplaycity.com -stopnarco.ru -stroyhelp-dv.ru -studentguide.ru -sukirgenk.dvrlists.com -supermodni.com.ua -takeprofitsystem.com -talant-factory.ru -taqywu51.soup.io -teenforporn.com -terrafootwear.us -thenetinfo.com -timetorelax.biz -tksn.ru -topmira.com -torrents.life +sockshare.net +softwaretrend.net +sovinsteel.ru +sstroy44.ru +stanthonyscatholicchurch.org +surgut.zrus.org +svetodiodoff.ru +taraz.xkaz.org +templates.franklinfire.co +tgtclick.com +theheroes.ru +thetoiletpaper.com +tiens2010.ru +tizanidineoral.blogspot.com +tkanorganizma.ru +touchmods.fr trk-4.net trucri.me -tula.mdverey.ru -twiclub.in -uhdtv.website -userequip.com -validdomain.xyz -velen.io -viagra.pp.ua -viktoria-center.ru -vipms.ru -vita.com.hr -voloslove.ru -voprosotvet24.ru -vykupavto-krasnodar.ru -vzglyadriv.kg -vzubah.com -w-journal.ru -webads.co.nz -webpromotion.ae -word-vorlagen.net -xdoza.com -xmronta.com -xn--1-8sbcpb0bdm8k6a.xn--p1ai -xn--80aeahghtf8ac5i.xn--p1ai -xn--b1agm2d.net -xn--l1aengat.xn--p1ai -xxxtubesafari.com -zhacker.net -znakomstva-moskva77.ru +ufa.xrus.org +uhodzalijami.ru +unimodemhalfduplefw.pen.io +universals.com.ua +vandrie-ict.nl +vbikse.com +vellings.info +viagra-soft.ru +viandpet.com +viel.su +vilingstore.net +vsexkontakte.net +weburlopener.com +wesharepics.site +white-truck.life +wurr.voila.net +x-diesel.info +x-lime.com +xkaz.org +xn-----6kccaibs5cb8afhjrfmix2n.xn--p1ai +xn----7sbahjd3btneuw1joc.xn--p1ai +xn--80aanaardaperhcem4a6i.com +xn--80ajbshivpvn2i.xn--p1ai +xn--e1aggki3c.xn--80adxhks +xsfetish.org +xvideospornoru.com +xxxhdvideo.site +youjizz.com +zaobao.com.sg +zero1.it +zootravel.com +zrus.org diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index f1c3ae34d..f3b3bcf9d 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.228 0; +10.1.0.47 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index e565d7cc5..7600c320b 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2021.08.2510 -### Updated: Sun Aug 8 08:39:08 UTC 2021 -### Bad Bot Count: 618 +### Version: V4.2021.08.2511 +### Updated: Sun Aug 8 09:36:43 UTC 2021 +### Bad Bot Count: 620 ################################################### ### Version Information ## @@ -195,6 +195,8 @@ User-agent: Cosmos Disallow:/ User-agent: Craftbot Disallow:/ +User-agent: Crawling at Home Project +Disallow:/ User-agent: CrazyWebCrawler Disallow:/ User-agent: Crescent @@ -1135,6 +1137,8 @@ User-agent: autoemailspider Disallow:/ User-agent: backlink-check Disallow:/ +User-agent: cah.io.community +Disallow:/ User-agent: check1.exe Disallow:/ User-agent: coccocbot-web diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index e565d7cc5..7600c320b 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -6,9 +6,9 @@ ### Version Information # ################################################### -### Version: V4.2021.08.2510 -### Updated: Sun Aug 8 08:39:08 UTC 2021 -### Bad Bot Count: 618 +### Version: V4.2021.08.2511 +### Updated: Sun Aug 8 09:36:43 UTC 2021 +### Bad Bot Count: 620 ################################################### ### Version Information ## @@ -195,6 +195,8 @@ User-agent: Cosmos Disallow:/ User-agent: Craftbot Disallow:/ +User-agent: Crawling at Home Project +Disallow:/ User-agent: CrazyWebCrawler Disallow:/ User-agent: Crescent @@ -1135,6 +1137,8 @@ User-agent: autoemailspider Disallow:/ User-agent: backlink-check Disallow:/ +User-agent: cah.io.community +Disallow:/ User-agent: check1.exe Disallow:/ User-agent: coccocbot-web