diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 97de7be88..710b0dfe0 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2486 +#### Version: V4.2021.07.2487 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [614](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index bf74e7290..ec5c70e5e 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2486 +#### Version: V4.2021.07.2487 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [614](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index 463412541..1896a4bdc 100755 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2021.07.2486 +#### Version: V4.2021.07.2487 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [614](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list index f5b19d942..11c78ad5a 100755 --- a/_generator_lists/bad-user-agents.list +++ b/_generator_lists/bad-user-agents.list @@ -1,4 +1,3 @@ -ALittle\ Client 01h4x.com 360Spider 404checker @@ -6,6 +5,7 @@ ALittle\ Client 80legs ADmantX AIBOT +ALittle\ Client ASPSeek Abonti Aboundex diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 58ca39f2d..f5eb50805 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2021.07.2486 +#### Version: V4.2021.07.2487 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [614](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index b20314d3d..2dcefa68f 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2486 -### Updated: Tue Jul 27 14:07:44 UTC 2021 +### Version: V4.2021.07.2487 +### Updated: Wed Jul 28 07:39:58 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 614 +### Bad Bot Count: 615 ################################################### ### VERSION INFORMATION ## @@ -154,6 +154,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)80legs(?:\b)" 3; "~*(?:\b)ADmantX(?:\b)" 3; "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)ALittle\ Client(?:\b)" 3; "~*(?:\b)ASPSeek(?:\b)" 3; "~*(?:\b)Abonti(?:\b)" 3; "~*(?:\b)Aboundex(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 9a0e001dd..2cc3d18a4 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2486 -### Updated: Tue Jul 27 14:07:45 UTC 2021 +### Version: V3.2021.07.2487 +### Updated: Wed Jul 28 07:39:58 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 614 +### Bad Bot Count: 615 ################################################### ### VERSION INFORMATION ## @@ -154,6 +154,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)80legs(?:\b)" 3; "~*(?:\b)ADmantX(?:\b)" 3; "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)ALittle\ Client(?:\b)" 3; "~*(?:\b)ASPSeek(?:\b)" 3; "~*(?:\b)Abonti(?:\b)" 3; "~*(?:\b)Aboundex(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index b20314d3d..2dcefa68f 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2486 -### Updated: Tue Jul 27 14:07:44 UTC 2021 +### Version: V4.2021.07.2487 +### Updated: Wed Jul 28 07:39:58 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 614 +### Bad Bot Count: 615 ################################################### ### VERSION INFORMATION ## @@ -154,6 +154,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)80legs(?:\b)" 3; "~*(?:\b)ADmantX(?:\b)" 3; "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)ALittle\ Client(?:\b)" 3; "~*(?:\b)ASPSeek(?:\b)" 3; "~*(?:\b)Abonti(?:\b)" 3; "~*(?:\b)Aboundex(?:\b)" 3; diff --git a/dev-tools/_robots_input/robots-input.txt b/dev-tools/_robots_input/robots-input.txt index 1eb04ca99..2f884078e 100644 --- a/dev-tools/_robots_input/robots-input.txt +++ b/dev-tools/_robots_input/robots-input.txt @@ -5,6 +5,7 @@ 80legs ADmantX AIBOT +ALittle Client ASPSeek Abonti Aboundex diff --git a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 9a0e001dd..2cc3d18a4 100644 --- a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2486 -### Updated: Tue Jul 27 14:07:45 UTC 2021 +### Version: V3.2021.07.2487 +### Updated: Wed Jul 28 07:39:58 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 614 +### Bad Bot Count: 615 ################################################### ### VERSION INFORMATION ## @@ -154,6 +154,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)80legs(?:\b)" 3; "~*(?:\b)ADmantX(?:\b)" 3; "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)ALittle\ Client(?:\b)" 3; "~*(?:\b)ASPSeek(?:\b)" 3; "~*(?:\b)Abonti(?:\b)" 3; "~*(?:\b)Aboundex(?:\b)" 3; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index ca7cdc872..50cf62a52 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.227 1; +10.1.0.76 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index a229fc66b..687d89316 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.227 0; +10.1.0.76 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 9a0e001dd..2cc3d18a4 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2486 -### Updated: Tue Jul 27 14:07:45 UTC 2021 +### Version: V3.2021.07.2487 +### Updated: Wed Jul 28 07:39:58 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 614 +### Bad Bot Count: 615 ################################################### ### VERSION INFORMATION ## @@ -154,6 +154,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)80legs(?:\b)" 3; "~*(?:\b)ADmantX(?:\b)" 3; "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)ALittle\ Client(?:\b)" 3; "~*(?:\b)ASPSeek(?:\b)" 3; "~*(?:\b)Abonti(?:\b)" 3; "~*(?:\b)Aboundex(?:\b)" 3; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index ca7cdc872..50cf62a52 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.227 1; +10.1.0.76 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index a229fc66b..687d89316 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.227 0; +10.1.0.76 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 9a0e001dd..2cc3d18a4 100644 --- a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2486 -### Updated: Tue Jul 27 14:07:45 UTC 2021 +### Version: V3.2021.07.2487 +### Updated: Wed Jul 28 07:39:58 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 614 +### Bad Bot Count: 615 ################################################### ### VERSION INFORMATION ## @@ -154,6 +154,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)80legs(?:\b)" 3; "~*(?:\b)ADmantX(?:\b)" 3; "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)ALittle\ Client(?:\b)" 3; "~*(?:\b)ASPSeek(?:\b)" 3; "~*(?:\b)Abonti(?:\b)" 3; "~*(?:\b)Aboundex(?:\b)" 3; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index ca7cdc872..50cf62a52 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.227 1; +10.1.0.76 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf index a2f11ef9b..d4d448d85 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -66,6 +66,7 @@ "~*(?:\b)80legs(?:\b|)" 0; "~*(?:\b)ADmantX(?:\b|)" 0; "~*(?:\b)AIBOT(?:\b|)" 0; + "~*(?:\b)ALittle\ Client(?:\b|)" 0; "~*(?:\b)ASPSeek(?:\b|)" 0; "~*(?:\b)Abonti(?:\b|)" 0; "~*(?:\b)Aboundex(?:\b|)" 0; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index a229fc66b..687d89316 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.227 0; +10.1.0.76 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 9a0e001dd..2cc3d18a4 100644 --- a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2486 -### Updated: Tue Jul 27 14:07:45 UTC 2021 +### Version: V3.2021.07.2487 +### Updated: Wed Jul 28 07:39:58 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 614 +### Bad Bot Count: 615 ################################################### ### VERSION INFORMATION ## @@ -154,6 +154,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)80legs(?:\b)" 3; "~*(?:\b)ADmantX(?:\b)" 3; "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)ALittle\ Client(?:\b)" 3; "~*(?:\b)ASPSeek(?:\b)" 3; "~*(?:\b)Abonti(?:\b)" 3; "~*(?:\b)Aboundex(?:\b)" 3; diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index a385b0c6e..408e4f9ca 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -2486 +2487 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index 9a0e001dd..2cc3d18a4 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2486 -### Updated: Tue Jul 27 14:07:45 UTC 2021 +### Version: V3.2021.07.2487 +### Updated: Wed Jul 28 07:39:58 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 614 +### Bad Bot Count: 615 ################################################### ### VERSION INFORMATION ## @@ -154,6 +154,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)80legs(?:\b)" 3; "~*(?:\b)ADmantX(?:\b)" 3; "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)ALittle\ Client(?:\b)" 3; "~*(?:\b)ASPSeek(?:\b)" 3; "~*(?:\b)Abonti(?:\b)" 3; "~*(?:\b)Aboundex(?:\b)" 3; diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index b20314d3d..2dcefa68f 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2486 -### Updated: Tue Jul 27 14:07:44 UTC 2021 +### Version: V4.2021.07.2487 +### Updated: Wed Jul 28 07:39:58 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 614 +### Bad Bot Count: 615 ################################################### ### VERSION INFORMATION ## @@ -154,6 +154,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)80legs(?:\b)" 3; "~*(?:\b)ADmantX(?:\b)" 3; "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)ALittle\ Client(?:\b)" 3; "~*(?:\b)ASPSeek(?:\b)" 3; "~*(?:\b)Abonti(?:\b)" 3; "~*(?:\b)Aboundex(?:\b)" 3; diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 207d50264..1a18a837b 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -13.89.50.82 0; +52.143.111.111 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index b20314d3d..2dcefa68f 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2486 -### Updated: Tue Jul 27 14:07:44 UTC 2021 +### Version: V4.2021.07.2487 +### Updated: Wed Jul 28 07:39:58 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 614 +### Bad Bot Count: 615 ################################################### ### VERSION INFORMATION ## @@ -154,6 +154,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)80legs(?:\b)" 3; "~*(?:\b)ADmantX(?:\b)" 3; "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)ALittle\ Client(?:\b)" 3; "~*(?:\b)ASPSeek(?:\b)" 3; "~*(?:\b)Abonti(?:\b)" 3; "~*(?:\b)Aboundex(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index ca7cdc872..50cf62a52 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.227 1; +10.1.0.76 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index a229fc66b..687d89316 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.227 0; +10.1.0.76 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index b20314d3d..2dcefa68f 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2486 -### Updated: Tue Jul 27 14:07:44 UTC 2021 +### Version: V4.2021.07.2487 +### Updated: Wed Jul 28 07:39:58 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 614 +### Bad Bot Count: 615 ################################################### ### VERSION INFORMATION ## @@ -154,6 +154,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)80legs(?:\b)" 3; "~*(?:\b)ADmantX(?:\b)" 3; "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)ALittle\ Client(?:\b)" 3; "~*(?:\b)ASPSeek(?:\b)" 3; "~*(?:\b)Abonti(?:\b)" 3; "~*(?:\b)Aboundex(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index ca7cdc872..50cf62a52 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.227 1; +10.1.0.76 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index a229fc66b..687d89316 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.227 0; +10.1.0.76 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index b20314d3d..2dcefa68f 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2486 -### Updated: Tue Jul 27 14:07:44 UTC 2021 +### Version: V4.2021.07.2487 +### Updated: Wed Jul 28 07:39:58 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 614 +### Bad Bot Count: 615 ################################################### ### VERSION INFORMATION ## @@ -154,6 +154,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)80legs(?:\b)" 3; "~*(?:\b)ADmantX(?:\b)" 3; "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)ALittle\ Client(?:\b)" 3; "~*(?:\b)ASPSeek(?:\b)" 3; "~*(?:\b)Abonti(?:\b)" 3; "~*(?:\b)Aboundex(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index ca7cdc872..50cf62a52 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.227 1; +10.1.0.76 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index a2f11ef9b..d4d448d85 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -66,6 +66,7 @@ "~*(?:\b)80legs(?:\b|)" 0; "~*(?:\b)ADmantX(?:\b|)" 0; "~*(?:\b)AIBOT(?:\b|)" 0; + "~*(?:\b)ALittle\ Client(?:\b|)" 0; "~*(?:\b)ASPSeek(?:\b|)" 0; "~*(?:\b)Abonti(?:\b|)" 0; "~*(?:\b)Aboundex(?:\b|)" 0; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index a229fc66b..687d89316 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.227 0; +10.1.0.76 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index b20314d3d..2dcefa68f 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2486 -### Updated: Tue Jul 27 14:07:44 UTC 2021 +### Version: V4.2021.07.2487 +### Updated: Wed Jul 28 07:39:58 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 614 +### Bad Bot Count: 615 ################################################### ### VERSION INFORMATION ## @@ -154,6 +154,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)80legs(?:\b)" 3; "~*(?:\b)ADmantX(?:\b)" 3; "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)ALittle\ Client(?:\b)" 3; "~*(?:\b)ASPSeek(?:\b)" 3; "~*(?:\b)Abonti(?:\b)" 3; "~*(?:\b)Aboundex(?:\b)" 3; diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 207d50264..1a18a837b 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -13.89.50.82 0; +52.143.111.111 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 680f34474..c76e18dd6 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2486 -### Updated: Tue Jul 27 14:07:44 UTC 2021 +### Version: V4.2021.07.2487 +### Updated: Wed Jul 28 07:39:58 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 614 +### Bad Bot Count: 615 ################################################### ### VERSION INFORMATION ## @@ -154,6 +154,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)80legs(?:\b)" 3; "~*(?:\b)ADmantX(?:\b)" 3; "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)ALittle\ Client(?:\b)" 3; "~*(?:\b)ASPSeek(?:\b)" 3; "~*(?:\b)Abonti(?:\b)" 3; "~*(?:\b)Aboundex(?:\b)" 3; diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 207d50264..1a18a837b 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -13.89.50.82 0; +52.143.111.111 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 274f5c58d..b5655795e 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2486 -### Updated: Tue Jul 27 14:07:44 UTC 2021 +### Version: V4.2021.07.2487 +### Updated: Wed Jul 28 07:39:58 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 614 +### Bad Bot Count: 615 ################################################### ### VERSION INFORMATION ## @@ -154,6 +154,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)80legs(?:\b)" 3; "~*(?:\b)ADmantX(?:\b)" 3; "~*(?:\b)AIBOT(?:\b)" 3; + "~*(?:\b)ALittle\ Client(?:\b)" 3; "~*(?:\b)ASPSeek(?:\b)" 3; "~*(?:\b)Abonti(?:\b)" 3; "~*(?:\b)Aboundex(?:\b)" 3; diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index ca7cdc872..50cf62a52 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.227 1; +10.1.0.76 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/blacklist-user-agents.conf b/dev-tools/test_units/blacklist-user-agents.conf index a2f11ef9b..d4d448d85 100644 --- a/dev-tools/test_units/blacklist-user-agents.conf +++ b/dev-tools/test_units/blacklist-user-agents.conf @@ -66,6 +66,7 @@ "~*(?:\b)80legs(?:\b|)" 0; "~*(?:\b)ADmantX(?:\b|)" 0; "~*(?:\b)AIBOT(?:\b|)" 0; + "~*(?:\b)ALittle\ Client(?:\b|)" 0; "~*(?:\b)ASPSeek(?:\b|)" 0; "~*(?:\b)Abonti(?:\b|)" 0; "~*(?:\b)Aboundex(?:\b|)" 0; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 800924532..358f33abd 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Blow -CherryPicker -CopyRightCheck -GetRight -JustView -Mr.4x3 -Nameprotect -Semrush -seobility -sysscan +AhrefsBot +GoZilla +HEADMasterSEO +Ltx71 +PictureFinder +SlySearch +Spanner +WeSEE +arquivo.pt +heritrix diff --git a/dev-tools/test_units/random-bots-for-test.list b/dev-tools/test_units/random-bots-for-test.list index 2c45f3100..4dfce44ba 100644 --- a/dev-tools/test_units/random-bots-for-test.list +++ b/dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ 01h4x.com -404checker -404enemy +360Spider 80legs -Aboundex -Acunetix -AlphaBot +ADmantX +AIBOT +ALittle Client +ASPSeek +AfD-Verbotsverfahren +AhrefsBot Anarchie -Apexoo -Attach -AwarioSmartBot -BBBike +Anarchy +Aspiegel +AwarioRssBot BDCbot BackDoorBot -BackStreet BackWeb -BacklinkCrawler -BatchFTP -Bitacle -Black Hole +Bandit +Battleztar Bazinga +Bigfoot +Blackboard Blow +BlowFish Boardreader -BotALot -Brandprotect -BuiltBotTough -CSHttp -Cegbfeieh +Buck +BuzzSumo +CCBot +CazoodleBot +CensysInspect CheTeam -CherryPicker -Chlooe -Claritybot +Cliqzbot Cloud mapping -Cogentbot -Copier +Collector +CopyRightCheck +Cosmos Craftbot -CrunchBot +CrazyWebCrawler DTS Agent +DatabaseDriverMysqli Demon Dirbuster -Disco -Dispatch -DnyzBot +DittoSpyder +DomCopBot DomainCrawler DomainSigmaCrawler -DomainStatsBot -Domains Project +Download Wonder Dragonfly -EMail Siphon +ECCP/1.0 +EMail Wolf EasyDL -Ecxi +EroCrawler Evil ExtLinksBot -Extreme Picture Finder -FHscan -Fimap +FDM Firefox/7.0 FlashGet -FrontPage +Freeuploader +Fuzz FyberSpider -Fyrebot +G-i-g-a-b-o-t GT::WWW GalaxyBot Getintent -Gigablast +Gigabot Go!Zilla -Go-Ahead-Got-It -GrabNet -GrapeFX -Haansoft -Heritrix +GoZilla +Gotit +Grafula +GrapeshotCrawler +HEADMasterSEO +HMView +HTTrack +Havij HybridBot -Iblog -Image Fetch +IDBot +Id-search +IlseBot +InfoNaviRobot InterGET +Internet Ninja +InternetSeer IstellaBot +JOC Web Spider JamesBOT JetCar -Jetty -Joomla -Jorgee +JikeSpider Jyxobot Keyword Density -LeechGet -LibWeb +Larbin +LexiBot LieBaoFast -Lightspeedsystems Likse -LinkScan -Linkbot -Linkdexbot -LinkextractorPro -LinkpadBot -Lipperhey +LinkWalker +LinqiaMetadataDownloaderBot Lipperhey Spider -Litemage_walker Lmspider -Ltx71 +MIDown tool +MIIxpc +MJ12bot Mag-Net Majestic SEO -MarkWatch -MeanPath Bot -MegaIndex.ru +Majestic-SEO +Majestic12 +MarkMonitor +Mass Downloader +Mata Hari +MauiBot +Mediatoolkitbot +MicroMessenger Microsoft Data Access Microsoft URL Control -Moblie Safari Mojeek -Mojolicious -Msrabot +Morfeus Fucking Scanner +Mr.4x3 +NICErsPRO +NPbot Nameprotect Navroad +NearSite +Nessus Net Vampire NetAnts -NetMechanic -NetSpider -Netcraft -NextGenSearchBot -Niki-bot -Nimbostratus -Nmap -Offline Navigator +NetZIP +Nettrack +NimbleCrawler +Ninja +Octopus +Offline Explorer OnCrawl -OpenVAS Openfind OrangeSpider -OutclicksBot PHPCrawl POE-Component-Client-HTTP +Page Analyzer +PageGrabber +PageScorer PageThing.com +Panscient Papa Foto +Pavuk PeoplePal -Picsearch -PictureFinder +Piepmatz Pimonster PleaseCrawl ProPowerBot Pu_iN -PxBroker PyCurl -Quick-Crawler RankActive -RankFlex -RankingBot +RankActiveLinkBot +RankingBot2 Rankivabot -RankurBot -RebelMouse -RedesScrapy -RepoMonkey -RocketCrawler +ReGet +Reaper +Ripper +SBIder SEOkicks -SEOkicks-Robot -SMTBot -SalesIntelligent -ScoutJet -Screaming -ScrepyBot +Scanbot +Scrapy +ScreenerBot Searchestate SearchmetricsBot -Semrush SeobilityBot Seomoz Shodan Siphon Site Sucker SiteExplorer -SiteLockSpider +SiteSucker Sitebeam Siteimprove -Sitevigil -SlySearch +Snapbot Snoopy -SocialRankIOBot -Sogou web spider -Sottopop -Spammen +Sosospider +SpaceBison +SpankBot SputnikBot -Sqlmap Sqlworm -Steeler -SuperBot +Stripper +SuperHTTP Surfbot -Suzuran Swiftbot -T8Abot Teleport +TeleportPro +Telesoft Telesphorep -TightTwatBot -Toweyabot -Tracemyfile -Twice -V-BOT +Thumbor +Toata +Trendiction +Trendictionbot +Turingos +TurnitinBot +TwengaBot +URLy Warning +UnisterBot Vacuum -Vagabondo VeriCiteCrawler -VidibleScraper -Virusdie -WBSearchBot +Voil +WASALive-Bot WEBDAV +WISENutbot WPScan -WWW-Collector-E WWW-Mechanize -WWW::Mechanize -Wallpapers/3.0 -Web Enhancer +Web Fetch Web Pix WebAuto WebBandit -WebCollage -WebEnhancer +WebCopier +WebFetch +WebFuck WebGo IS +WebImageCollector WebLeacher -WebPix +WebReaper +WebSauger WebStripper +WebSucker WebWhacker -WebZIP -WebsiteExtractor -WebsiteQuester -Webster -Whack -Whacker -Widow -Wonderbot -Woobot +Webalta +WebmasterWorldForumBot +Website Quester +Who.is Bot +WinHTTrack Wotbox +Wprecon +Xaldon WebSpider Xenu Zauba -Zermelo -Zeus -ZmEu -arquivo.pt +ZyBorg +backlink-check check1.exe cognitiveseo -com.plumanalytics -gopher +crawl.sogou.com +crawler4j +demandbase-bot +evc-batch heritrix -instabid -ips-agent lwp-trivial -meanpathbot -netEstate NE Crawler -page scorer plumanalytics +polaris version +probe-image-size +scalaj-http scan.lol -siteripz +seobility +seostar +serpstatbot sogouspider -trendiction.de +tAkeOut voyagerx.com x09Mozilla x22Mozilla zauba.io -zgrab diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 2a8970e1a..cdd0a1d60 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -360Spider 404checker 80legs +AIBOT +ALittle Client ASPSeek +Abonti +Aboundex +Aboundexbot Acunetix AfD-Verbotsverfahren +AhrefsBot +AiHitBot Aipbot -AlphaBot -Ankit -Aspiegel +Alexibot +Anarchie +Anarchy +Anarchy99 +Anthill Asterias Attach +AwarioSmartBot BBBike -BDCbot +BackDoorBot BackStreet -Badass -Barkrowler +BacklinkCrawler +Bandit +BatchFTP +Bigfoot Bitacle -Black Hole -BlackWidow -Boardreader -Buddy -BuiltWith -BunnySlippers +Blow +Brandprotect +Brandwatch +Bullseye BuzzSumo -CODE87 -CSHttp +CazoodleBot +CensysInspect +CheTeam CheeseBot -Cloud mapping +ChinaClaw +Chlooe +Claritybot Cocolyzebot -CopyRightCheck +Cogentbot +Copier +Craftbot CrazyWebCrawler -CrunchBot Curious -DataCha0s -Demon -Digincore -Disco +Custo +DBLBot +DSearch +Devil +Discoverybot Dispatch -DnyzBot -DomainSigmaCrawler +DomCopBot +DomainAppender +DomainCrawler DomainStatsBot -Domains Project -Drip +ECCP/1.0 EMail Siphon -EasyDL -Ebingbong Ecxi -EirGrabber -Evil Express WebPictures -ExtLinksBot Extreme Picture Finder FDM +FHscan +FemtosearchBot Freeuploader FyberSpider Fyrebot Genieo -Getintent -GrabNet -Grabber +GetWeb +Gotit Grafula -GrapeshotCrawler -HTTP::Lite -Harvest -Heritrix +HEADMasterSEO +Haansoft +Havij Hloader -Humanlinks -HybridBot -Iblog +IDBTE4M Id-search -Image Fetch -Image Sucker -IndeedBot InfoNaviRobot InfoTekies InterGET -Internet Ninja -IstellaBot -JamesBOT +InternetSeer +Iskanie +JOC Web Spider +Jbrofuzz +JennyBot +JetCar +Jetty JikeSpider Joomla -JustView -Kenjin Spider +Keyword Density Kinza +LWP::Simple Lanshanbot Larbin -LeechFTP -LexiBot Lftp -LieBaoFast -LinkScan +Libwhisker +LinkWalker Linkbot -LinqiaMetadataDownloaderBot +Linkdexbot +LinkpadBot +LinksManager LinqiaScrapeBot -Lipperhey -MQQBrowser +Lipperhey Spider +Lmspider +Ltx71 +MFC_Tear_Sample +MJ12bot MSIECrawler -MTRobot -Magnet Mail.RU_Bot -Majestic SEO -Majestic12 -MarkMonitor +Majestic-SEO MarkWatch -MauiBot +Masscan +Mata Hari +MeanPath Bot Meanpathbot Metauri MicroMessenger -Microsoft Data Access -Microsoft URL Control +Mister PiX Moblie Safari -Mojolicious -Morfeus Fucking Scanner -Nameprotect -NearSite -Needle -Nessus +Mojeek +Mr.4x3 +Msrabot +NPbot +Name Intelligence Net Vampire -NetAnts +NetLyzer +Netcraft +Nettrack +Netvibes +Nibbler +Nikto +NimbleCrawler Nimbostratus Nmap -Nuclei -Octopus -Offline Explorer +Nutch OnCrawl -OrangeBot -OutfoxBot +Openfind +Openvas PECL::HTTP -Page Analyzer -PageAnalyzer -Pandalytics Papa Foto -Petalbot -Pi-Monster -Picscout +Pavuk +PeoplePal PictureFinder -Piepmatz +Pimonster +Pixray +PleaseCrawl Pockey -ProWebWalker +Probethenet Pu_iN -PyCurl -RankActiveLinkBot +PxBroker +QueryN Metasearch +RSSingBot +RankActive RankingBot2 +ReGet RealDownload -Recorder -RedesScrapy -Rogerbot +RebelMouse +RepoMonkey SEOkicks -SEOstats +SMTBot +SalesIntelligent Screaming -SearchmetricsBot -Semrush +SemrushBot SentiBot -SeoSiteCheckup -Seomoz Shodan -Siphon Site Sucker +SiteCheckerBotCrawler SiteExplorer SiteLockSpider -SiteSnagger -SiteSucker +Sitebeam Siteimprove -SmartDownload -Sociscraper +Sitevigil +SlySearch +Snoopy +SocialRankIOBot Sottopop +SpaceBison +SpankBot +Spinn3r SputnikBot -Sqworm +Sqlworm Steeler -Stripper -SuperBot -SuperHTTP -SurveyBot +Sucuri +Surfbot Suzuran -Swiftbot -T8Abot -Teleport +Szukacz +TeleportPro Telesoft Telesphoreo Telesphorep The Intraformant -Titan -Toata +TightTwatBot Tracemyfile Trendictionbot -True_Robot Turnitin TurnitinBot -UnisterBot -VCI +Upflow +V-BOT Vacuum +VeriCiteCrawler Voil Voltron -WASALive-Bot WBSearchBot -WEBDAV -WISENutbot -WPScan -Web Auto -Web Fetch +WWW-Mechanize +WWWOFFLE +WallpapersHD +Web Collage +Web Enhancer Web Fuck -Web Sucker WebBandit -WebCollage -WebReaper -WebSauger -WebStripper -WebSucker +WebEnhancer +WebImageCollector +WebPix WebWhacker WebZIP Webalta WebmasterWorldForumBot -WebsiteQuester -Whack -Who.is Bot +Webster +Whacker +Whatweb Widow -WinHTTrack -WiseGuys Robot +Woobot Wotbox -Wprecon Xaldon WebSpider -YoudaoBot -Zade Zauba Zeus -ZmEu ZyBorg archive.org_bot -arquivo.pt check1.exe +coccocbot-web cognitiveseo -crawl.sogou.com -domainsproject.org -heritrix +com.plumanalytics +crawler.feedback +demandbase-bot +evc-batch internetVista monitor -isitwp.com -lwp-request -mediawords -muhstik-scan -page scorer -plumanalytics -probe-image-size -scan.lol +meanpathbot +netEstate NE Crawler +polaris version +satoristudio.net +scalaj-http seobility -seostar -sitechecker.pro +sogouspider sp_auditbot -tAkeOut +voyagerx.com +webmeup-crawler +x22Mozilla xpymep1.exe +zauba.io diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 1c14ae00d..a485864af 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -businescoop.men -calstaterealty.us -christianlouboutinsaleoutletonline.us -mediafresh.online -myiptest.com -online-x.ru -photosaga.info -riralmolamsaca.tk -suture.co -vut.com.ru +brewdom.ru +houtings.xyz +mansparskats.com +rewardpoll.com +sdsjweb.com +sevenstars7.com +sex-watch.com +sexs-foto.top +xn--80ab4aa2g.xn--p1ai +zerocash.msk.ru diff --git a/dev-tools/test_units/random-referrers-for-test.list b/dev-tools/test_units/random-referrers-for-test.list index 31678dd84..a045fbfef 100644 --- a/dev-tools/test_units/random-referrers-for-test.list +++ b/dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ +08800.top +1000n1.ru +123cha.com 12bet.com -12masterov.com -3rbseyes.com -3xforum.ro -8lufu.com -abcguru.xyz -acmebtn.ml -activecampaign.dreamhosters.com -addtoadd.men -adman.gr -adserver-e7.com -adultmeetonline.info -affiliate-fr.com -allornamenti.com -alphacarolinas.com -amospalla.es -ananumous.ru -animalrank.com -anticrawler.org -apartamentwroclaw.eu -apiportalmorecom-a.akamaihd.net -arabgirls.adultgalls.com -arewater.com -arquapetrarca.info -artavenuegardenstudios.com -artel116.ru -arvut.org -asseenontv.ru -autochoixspinelli.com -autogrep.ru -autoloans.com -avramstroy.ru -avto-oligarh.ru -azartmsl.com -backuperwebcam.weebly.com -baditri.com -baladur.ru -bang-hotties.com -bayareaaeroclub.org -beachdriveblog.com -best-businessman.ru -bestcurs.org -bestempresas.es -bestmouthwash.club -bestpriceninja.com -bestwaystogetpaid.us -bikini-image.com -bkgr.se -blacktwhite.com -blavia.00author.com -bongacams.com -bonzibuddi.com -bracketsmackdown.com -bradleylive.xyz -brainzpod.win -brandbucket.com -brandedleadgeneration.com -brandehk.dk -bugof.gq -buqayy0.livejournal.com -businesn.men -buttons-for-your-website.com +12u.info +1kinobig.ru +404.mn +5elementov.ru +5kstore.com +8si.ru +aa14ab57a3339c4064bd9ae6fad7495b5f.com +aac52006ec82a24e08b665f4db2b5013f7.com +aaronabel.com +abctoppictures.net +absugars.com +abwa.tk +acgs.tk +activepr.ru +adanih.com +adloads.com +adservme.com +adsloads.com +adultfriendfinder.com +advertisingtag.net +adzpower.com +agadelha.com.br +agroeconom.kz +ahrntal.verymes.xyz +aimayangzhi.com +alert-fjg.xyz +alevinefesleri.com +alfa-img.com +all4wap.ru +allwidewallpapers.com +alpinism.ru +amateurgalls.com +amazingpic.net +angel1777.kz +animal-drawings.com +apartmentratings.com +apiluckyleapnet-a.akamaihd.net +apimountainbikei-a.akamaihd.net +app-ready.xyz +arabsexxxtube.com +arenanews.com.ua +arendakvartir.kz +arpelsreplica.xyz +artdestshop.eu +articlesdirectoryme.info +artpicso.com +asacopaco.tk +atlant-auto.info +atmovs.com +audiofree.ru +aussie-prizes.com +avtovolop.ru +ayeartoforget.com +backiee.com +bangla-chat-uk-paralud.ga +bankmib.ru +belinvestdom.by +best-coupon-offer.com +bestadbid.com +bestserials.com +biketank.ga +bizfly.info +biznesrealnost.ru +bluesman.nu +boole.onlinewebshop.net +braindaily.xyz +brillianty.info +budilneg.xyz +bumble.cheapwebsitehoster.com +burger-tycoon.com +buscarfoto.com +businescoop.men +businesxxl.com +buy-cheap-pills-order-online.com buynorxx.com -bzero1jewelry.net -cakemediahq.com.au -callejondelpozo.es -calstaterealty.us -cand.jp -cartierlovestore.com -cartierreplica.pw -cartierring.xyz -cbcseward.com -celejihad.info -centre-indigo.org.ua -cfacarrosserie74.com -cheap-pills-norx.com -cheatcode-lita12.rhcloud.com -chelnytruck.ru -chidporn.com -classicakuhni.ru -cleanersoft.com -clicksotrk.com -clothesforcash.com -club-musics.ru -com-onlinesupport.host -compliance-nelson.top -compliance-willy.top -compucelunlock.net -confib.ifmo.ru -copblock.org -copenergo.ru -corta.co -costablue.xyz -cougfan.info -crazyboost.pro -crazyprotocol.com +buypillsorderonline.com +buzzurl.jp +cablecar.us +calcularpagerank.com +candy-glam-hp.com +cardsdumps.com +cardsharp1.ru +catalogs-parts.com +cbozhe.com +cdncash.org +chatroulette.world +chcu.net +cheap-trusted-backlinks.com +cheapjerseysfootballshop.com +check-this-out-now.online +chee-by.biz +chinese-amezon.com +christianlouboutinreplica.win +cienum.fr +clarithromycin500mg.com +claytransformations.info +cleanallspyware.com +cntravelre.com +codq.info +com-supportcenter.website +compliance-alex.top +compliance-ivan.xyz +compliance-john.top +compliance-julianna.top +connectionstrenth.com +contextualyield.com +cookie-law-enforcement-ff.xyz +coolbar.pro +counter.yadro.ru +creativehutindia.com creditwell.ru -crystalslot.com -cwetochki.ru -dailystrength.org -date-withme.com -daymusam.com -deadlinkchecker.com +custom-electric-guitar.com +d-black.bz +danashop.ru dealitright.click -decorazilla.com -delayreferat.ru -deluxedumps.com -dentfidemountpreach.tk -derevesendeco.com -dev.citetick.com -diffbot.com -digitalfaq.com -divisioncore.com -dnepropetrovsk.mistr-x.org -dneprsvet.com.ua -domain-submit.info -domcran.net -dostavka-v-ukrainu.ru -dosug-lux.ru +designdevise.com +destinationrealestate.com +detskie-konstruktory.ru +discovertreasurenow.com +dokumentalkino.net +domashneeruporno.com download-walpaperhd.blogspot.com -downloadmefiranaratb1972.xpg.com.br -drugspowerstore.com -dstroy.su -duawitchrarato.tk -dumuelave.xyz -e90post.com -ebonyporn.site -ecommerce-seo.com +dptaughtme.com +droidlook.net +drugstoreforyou.com +duplicashapp.com +easycommerce.cf egovaleo.it -el-nation.com -elektrozigaretten2.yn.lt -eluxer.net -emailaccountlogin.co -emergencyneeds.org -empathica.com -enskedesquashclub.se -erotag.com -erotik-kostenlos.net -erotikchat-24.com -evaashop.ru -exchanges-bet.com -expertblog.info -exportshop.us -extrabot.com -extremepornos.net -extremez.net -eywords-monitoring-your-success.com -ezigarettenshop2.myblog.de -f1nder.org -fatfasts-4tmz.com -fatmaelgarny.com -favoritemoney.ru -femmesdenudees.com -ff30236ddef1465f88547e760973d70a.com -fickenprivat.info -filmbokep69.com -financetip.eu -fishingwholesale.us -forensicpsychiatry.ru -formulaantiuban.com -forums.d2jsp.org +elektrozigaretten1.postbit.com +eloconcream.blogspot.com +elstal.com.pl +encodable.com +eorogo.top +epicbrogaming.com +ero-advertising.com +eropho.com +event-tracking.com +extener.org +eyelike.com.ua +ez8motelseaworldsandiego.com +ezigarettekaufen.myblog.de +familienzahnaerzte.com +fast-torrent.ru +ferrotodo.com +filesmonster.porn +filesvine.com +film-one.ru +filmania-x.ru +findacheaplawyers.com +fingerprintjs.com +finteks.ru +fireads.men +foodgid.net +footballfarrago.com +forex-procto.ru +formulaf1results.blogspot.com +forodvd.com fotoxxxru.com -foxydeal.com -frameimage.org -funcrushgames.com -galaxys6manual.info -gameplexcity.com -gamezblox.com -gate5.co.za -gay-site.store -gembird.com -geoads.com -germes-trans.com -get-free-traffic-now.com -getaclueamerica.com -getoutofdebtfree.org -given2.com -glcomputers.ru -global-ics.co.za -glogow.pl -golfresa.lucania.se -gombita.info -goodwinmetals.co -goodwriterssales.com -gosreg.amchs.ru -gox.com.ua -graphicwe.org +foxinsocks.ru +freenode.info +freevpn.space +freewareseek.com +frvo.alptandem.ru +fxtips.ru +game-mmorpg.net +gamevalue7.weebly.com +gavuer.ru +gg-arena.ru +goatse.ru +gobongo.info +golden-catalog.pro +golden-praga.ru +gourcy.altervista.org +greenshop.su growmyfunds.ca -gsasearchenginerankerhelp.com -gsmlab.pl -handicapvantoday.com -hanink.biz.ly -hdapp1008-a.akamaihd.net +grupografico-pilar.com.ar +gsbs.com.ua +gtopstats.com +guildebzh.info +gz2.bbsoldes.fr +hasshe.com hdwallpapers-free.com +healing-dysplasia.ru heatpower.ru -highspeed5.net -histats.com -hotblog.top -hoterika.com +hermesbelts.xyz +hiwibyh.bugs3.com +hobbyhuren24.net +hollywoodactress.info +hostnow.men +hostsshop.ru hotgirlhdwallpaper.com -hothor.se -https-legalrc.biz +hothot.ru +houseofrose.com +hoverboard360.nl +htmlcorner.com +hulfingtonpost.com hut1.ru -i-midias.net.br -id-forex.com -ideoworld.org -impisr.edunsk.ru -inboundlinks.win -infobanks.ru -insta-add.pro -instakink.com -intim-uslugi.info -invest-pamm.ru -iptool.xyz -izhevsk.xrus.org -jackpotchances.com -javatex.co.id -javitas.info -javstock.com -jerseychinabizwholesale.com -jillepille.com -johannesburgsingles.co.za -kaac.ru -kakadu-interior.com.ua -kaliningrad.zrus.org -karusel-market.ru -kazinogames.lv -keyhantercume.com -keywesthideaways.co -keywordhouse.com +hyipmanager.in +idealtits.net +ifmo.ru +ihtec2019.org +ilovevitaly.ru +imagine-ex.co +importchinacoach-teach.com +indo-export.ru +iphantom.com +iscblog.info +istmira.ru +itag.pw +ix20.ru +izismile.com +jackwolfskinoutlet.online +japan-bearings.ru +japfm.com +jav-way.site +javxxx18.com +jerseywholesalebizchina.com +jerseywholesalechinabiz.com +jerseywholesaleelitestore.com +jimmychoosale.online +jjbabskoe.ru +jmat.cn +jurajskie.info +justprofit.xyz +jyvopys.com +kemerovo.zrus.org +kerwinandcariza.com kicknights.gq -kidskunst.info -kladrus.ru -klumba55.ru -kol-energo.ru -koptims.tiu.ru -kostenlos-sexvideos.com -krynica.info +kinocccp.net +kletkimehan.ru +kochanelli.com +koopilka.com +kovesszucs.atw.hu +krassh.ru +kredit-pod-zalog-krasnodar.ru ktotut.net -labplus.ru -lapolis.it -letmacworkfaster.site -lflash.ru -lider-zhaluzi.kiev.ua -lockerz.com -longlifelomilomi.info -louboutinreplica.pw -louisvuittonoutletstore.net -lsex.xyz -lutherstable.org -magicalfind-a.akamaihd.net -mapquestz.us -marcogrup.com -marcowebonyodziezowe.pl -marwer.info -maximpartnerspr.com -medizinreisen.de -medmajor.ru -megavolt.net.ua -memberty.com +kursy-ege.ru +landscaping.center +larger.io +larose.jb2c.me +law-check-nine.xyz +law-check-two.xyz +lifebyleese.com +likrot.com +linkarena.com +linkredirect.biz +liquimondo.com +liumimgx.pw +livingroomdecor.info +lookingglassemb.com +love-baby.cz +luciddiagnostics.in +m4ever.net +mac-shield.com +macdamaged.tech +machicon-ueno.info +mackeeper-center.club +mafcards.ru +magiadinamibia.blogspot.com +manifestyourmillion.com +mansiondelrio.co +manve.info +marblestyle.ru +medi-fitt.hu +meget.co.za +meltwater.com +mere.host.sk mesbuta.info -mexicosleevegastrectomy.com -mindbox.co.za -mistr-x.org -mobplayer.ru -mobstarr.com -mouselink.co +metabar.ru +minyetki.ru +missclub.info +mncrftpcs.com +mobot.site +moncleroutletonline.xyz +moneyteam24.com +moscow.xrus.org moviemail-online.co.uk -moyaterapiya.ru -mozello.ru -mriyadh.com -mustwineblog.com +mp3downloadhq.com +mrsdalloways.com +mrvideospornogratis.xxx +msk.afora.ru +musicvidz.ru myftpupload.com -mymobilemoneypages.com -myprintscreen.com -mzdish.site -na15.ru nagdak.ru -naturalbreakthroughsresearch.com -news-speaker.com -newstaffadsshop.club -newstraveller.ru -newtechspb.ru -newyorkhotelsmotels.info -nl.netlog.com -nodup.ru +nakozhe.com +narkologiya-penza.ru +national-today-winning-winner.club +needtosellmyhousefast.com +netvouz.com +networkcheck.xyz +nextbackgroundcheck.gq +nightvision746.weebly.com +niroo.info +nlfjjunb5.ru notasprensa.info -noumeda.com -novosibirsk.xrus.org -nowtorrents.com -o-o-8-o-o.ru -obiavo.ru -obnal.org +nuker.com +oakleyglassesonline.us +oakridgemo.com +obesidadealgarve.com ochistka-stokov.ru -oecnhs.info -offf.info +odywpjtw.bloger.index.hr +offtopic.biz +ohmyrings.com +oil-td.ru +oklogistic.ru +onemactrckr.com oneminutesite.it -onescreen.cc -oneshotdate.com -onko-24.com -online-podarki.com -online.ktc45.ru -onstrapon.purplesphere.in +ooo-gotovie.ru +oops-cinema.ru osagonline.ru -paceform.com -pateaswing.com +ossmalta.com +ownshop.win +paidonlinesites.com +pammik.ru +paramountmarble.co.uk pdns.download -pendelprognos.se -perfection-pleasure.ru -pharmacyincity.com -phormchina.com -photochki.com -photokitchendesign.com -phrcialiled.com -picsfair.com -pinup-casino1.ru -piuminiita.com -pixshark.com -pizza-tycoon.com -pkr1hand.com -play-mp3.com -play.leadzupc.com +pekori.to +picmoonco.pw +pila.pl +pixell.club +playlott.com playtap.us -pod-muzyku.club -polybuild.ru -popmarker.com -poppen-nw.net -poptool.net -pornoblood.com -pornofeuer.com -pornohd1080.online -pornosexrolik.com -pornotubexxx.name -pornozhara.com -portadd.men -pricheskaonline.ru -printie.com -prod2016.com -prodess.ru -projefrio.com.br -prostitutki-astana.org -prostitutki-yaroslavlya.org -psoriasis-file.trade -qexyfu.bugs3.com -quangcaons.com -rainbowice.ru -rank-checker.online -ranksonic.info -raschtextil.com.ua -rcpmda.ikan1080.xyz -reeyanaturopathy.com -remmling.de -remont-ustanovka-tehniki.ru -rique.host.sk -ruscoininvest.company -rusenvironmental.net -ruspdd.com -russia-tao.ru -rustic-quiver.win -ryetaw.com -sady-urala.ru -saitevpatorii.com -sarf3omlat.com -scanner-donald.top -scanner-nelson.top -scanspyware.net -scottbywater.com -searchimage.co -seo-prof1.xyz -seofied.com -seoprofiler.com -sevendays.com.ua -sex-pr.net -sex-sex-sex5.com -sexad.net -sexyali.com -sexystrippe.info -sexyteens.hol.es -sexytrend.ru -shemalegalls.blogporn.in -sibvitr.ru -signal03.ru -silvermature.net +polimga.pw +pontiacsolstice.info +popup-fdm.xyz +popup-hgd.xyz +porn-w.org +porno-chaman.info +pornolook.net +pornosmola.info +pornstartits.xblog.in +porodasobak.net +postclass.com +privatbank46.ru +prostitutki-kharkova.org +prostitutki-tyumeni.org +quebec-bin.com +queerspace.com +qwesa.ru +r-control.ru +rapevideosmovies.com +realtytimes.com +redirectme.net +rednise.com +rentaremotecomputer.com +replica-watch.ru +retreatia.com +reward-survey.net +ribieiendom.no +riralmolamsaca.tk +rome2rio.com +ros-ctm.ru +rosbalt.com.ua +rubbed.us +running-line.ru +runstocks.com +s.lollypopgaming.com +sajatvelemeny.com +sammlungfotos.online +samoiedo.it +sankt-peterburg.nodup.ru +santasgift.ml +sape.top +saratov.xrus.org +satoristudio.net +sbdl.no +scanner-fred.top +seadragonherbery.com +sei80.com +seksvideoonlain.com +sendearnings.com +seoexperimenty.ru +seozoom.it +serialsway.ucoz.ru +seryeznie-znakomstva.ru +sex-znakomstva.online +sex.hotblog.top +shariki-zuma-lines.ru +shillyourcoins.com +shoesonlinebuy.cn +sibdevice.ru simple-share-buttons.com -sindragosa.comxa.com -skatestick.bid -sklad-24.ru -skutecznetabletkinaporostwlosow.pl -slowmac.tech -slujbauborki.ru -smart-balancewheel.com -smartbalanceworld.com -smartpet.ru -snegozaderzhatel.ru -social-button.xyz -socialseet.ru -socialtrade.biz -sockshare.net -softomix.com -sophang8.com -sortthemesitesby.com +sitevaluation.com +sixcooler.de +skladvaz.ru +skuteczna-dieta.co.pl +sms2x2.ru +snjatie-geroinovoy-lomki.ru +soc-econom-problems.ru +songplanet.ru sotechco.co -sotkal.lark.ru -spitfiremusic.com -startwp.org -starwars.wikia.com -steamoff.net -stockspmb.info -strana-krasoty.ru -sucsesofinspiration.com -sukarame.net -superiends.org -surintech.ac.th -svarkagid.com -szamponrevita.pl -taaaak.com -tabakur77.com -tehngr.ru -thecounter.com -thexart.club -thfox.com -timeallnews.ru -tivolibasket.it -tizanidineandcipro.blogspot.com -tizanidineandhydrocodone.blogspot.com -tizanidinecapsules.blogspot.com -tizanidinedruginteractions.blogspot.com -tizanidinehcl2mg.blogspot.com -tizanidinerecreationaluse.blogspot.com -token-lab.org -toolsky.com -tootoo.to -top-l2.com -topflownews.com -topvidos.ru -trackzapper.com -trafficshaper.com -traffixer.com +sovetogorod.ru +spabali.org +spywarenuker.com +stardevine.com +stauga.altervista.org +staynplay.net +stillmiracle.com +supermama.top +supermesta.ru +swimpool.ca +takeflyte.com +tastyfoodideas.com +telefonsexi.com +the-trader.net +theprofitsmaker.net +titelhelden.eu +tizanidine4mgstreetprice.blogspot.com +tizanidine4mgstreetvalue.blogspot.com +tizanidinefibromyalgia.blogspot.com +tizanidineinteractions.blogspot.com +tizanidinemusclerelaxant.blogspot.com +tizanidineoral.blogspot.com +tjkckpytpnje.com +tnctrx.com +tonivedu.it +toon-families.com +toonfamilies.net +top-deal.com.pl +tour-line.net +track2.shop +tracksz.co +traffic2money.com +trafficmania.com +traffmonster.info +treasuretrack-a.akamaihd.net trkur.com -trustl.life -turist-strani.ru -turn-up-life.life -twinderbella.com -twittrading.com -uamtrk.com -ucoz.ru -ugguk.online -unrealcommander.org -url-extractor.xyz -v-doc.co -valuado.com -vduplo.ru -veselokloun.ru +trustedhealthtips.com +tt-ipd.info +tuberkuleznik.ru +tx41tclega.ru +u-cheats.ru +ukrtvir.com.ua +ukrtvory.in.ua +unece.org +uni.me +upstore.me +vancleefreplica.pw +vbabule.net +vbtracker.net +vegascosmetics.ru vezuviy.su -video--production.com -videogamesecrets.com -videokrik.net -vielporno.net -vietimgy.pw -villakohlanta.nu -vip-file.com -vip2ch.com -vladimir.xrus.org -volgograd.xrus.org -voloomoney.com -voloslove.ru -vut.com.ru -vzlomsn.org +viagra.pp.ua +video-chat.in +video-woman.com +virus-respirators.com +vitalads.net +vksex.ru +vladimir.zrus.org +voloo.ru +voprosotvet24.ru +vremya.eu +vse-pesni.com +vuclip.com +vykup-avto-krasnodar.ru +vzubah.com +wahicbefa31.soup.io +wareseeker.com watracker.net -wdrake.com -web-analytics.date -webshoppermac.com -website-stealer.nufaq.com -welcomeauto.ru -westen-z.life +webcamdevochka.com +website-analytics.online whatsappdownload10.com -whengirlsgowild.com -where-toget.com -while.cheapwebsitehoster.com -win-spy.com +wheelchairliftsarea.com +wholesalenfljerseys.us.com +wieseversa.no +wiosenny-bon-1500.pl +woman-tampon.ru +womensterritory.ru works.if.ua -worldmusicfests.com -wormix-cheats.ru -www.solartek.ru -wwwadultcheck.com +wovis.site wygraj-teraz.com -xn--80aaagvmjabrs1aoc9luc.xn--p1ai -xn--lifehacer-1rb.com -yebocasino.co.za -yourads.website -ypmuseum.ru -yuarra.pluto.ro -yunque.pluto.ro -zarenica.net -zeroredirect12.com -zigarettenonlinekaufen2.drupalgardens.com -zlatnajesen.com -zooggames.com -zs2vm.top -zytpirwai.net +x-diesel.org +x-mix.info +xbaboon.com +xblog.in +xboxster.ru +xnxx-n.com +xolodremont.ru +xxx-cam.webcam +xxxguitars.com +xxxsiterips.xyz +xxxtubesafari.com +yes-com.com +yoshkarola.zrus.org +youghbould.wordpress.com +yourothersite.com +youthreaders.com +youtoner.it +zaapplesales.blogspot.com +zdesoboi.com +zeroredirect6.com +znakom.sibtest.ru +zoodrawings.com +zoominfo.com +zrus.org +zryydi.com diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index 62421e784..92d5331fe 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -1stat.ru -2015god.org -2728fb936f0.com -4istoshop.com -7a2rnuey1tw9ar.ru -abctoppictures.net -adanih.com -adrunnr.com -agahinameh.com -alekseevec.ru -allcredits.su -amyfoxfitness.com -analnoeporno.tv -apibetweenlinesn-a.akamaihd.net -apiwoodensealcom-a.akamaihd.net -appfastplay.com -arius.tech -arquivo.pt -autoplate.info -autoseo-traffic.com -autoua.top -aviav.co -avon-ukraine.com -azbukafree.com -bestfortraders.com -bestkfiledxd.cf -beyan.host.sk -bobba.dzaba.com -bradleylive.xyz -brandehk.dk -briomotor.co -btnativenav.com -buyfriend.ru -c1.onedmp.com -cablecar.us +101flag.ru +1314dh.com +1qingdao.com +4pp13.com +5000-cotydzien.com +5ws.dating-app.ru +6hopping.com +7xc4n.com +abbanreddy.com +abtasty.com +adclickthru.net +addbin.men +adssafeprotected.com +adult3dgames.com +airlandsea.info +apccargo.com +apispringsmartne-a.akamaihd.net +apple.com-webbrowsing-security.science +apps-analytics.net +appsecurityr.com +arquapetrarca.info +avek.ru +avramstroy.ru +bestadbid.com +bestsexyblog.com +binomo.com +bio-optomarket.ru +biznesrealnost.ru +bkgr.se +blackhatworth.com +bocoarchives.org +bookhome.info +braindaily.xyz +builtwith.com +businesn.men +buyparajumpers.online +buzzsumo.com +c-gainsbourg.com +cactussoft.biz cam-kontakte.org -camsex.xxx-cam.webcam -carina-sy.de -cbozhe.com -chatroulette.video -classicakuhni.ru -colobit.biz -compliance-checker-7.info -compliance-walter.top -computernetworksonline.com -converse.ddsoldes.fr -copro.pw -cornomase.win -coslab.club -cpabegins.ru +candelluxsklep.pl +cardsdumps.com +cartierjusteunclou.xyz +cartierwatch.xyz +cenoval.ru +chinese-amezon.com +cialis-samples.com +club-samodelkin.ru +co.lumb.co +com-onlinesupport.host +compliance-josh.top +connectingsingles.com +consorzioilmosaico.org creditnation.ru -customcatchcan.com -cyprusbuyproperties.com -daooda.com -dfwu1019.info -diarioaconcagua.com -diffbot.com -dnepr-avtospar.com.ua -dood.live -dosug-lux.ru -doublepimp.com -doxysexy.com -earl-brown.info -el-nation.com -elkacentr.ru -enpolis.ru -exmasters.com -exonline.info +crest3d.ru +custodieva.ru +customchocolate.business-for-home.com +dbmkfhqk.bloger.index.hr +de.zapmeta.com +decoratinghomes.ga +depositfiles-porn.ga +digilander.libero.it +digitalfaq.com +dinkolove.ya.ru +djonwatch.ru +dneprsvet.com.ua +dnsrsearch.com +drev.biz +emediate.eu +engines-usa.com +es5.com +europages.com.ru +exchangeit.gq +expertblog.info exportshop.us -familyholiday.ml -fefo.gdn -fetlifeblog.com -ff30236ddef1465f88547e760973d70a.com -fidelityfunding.com -finansov.info -finemanteam.com -fireads.men -fodelsedagspresenter.nu -franch.info -fxgallery.com -gamewrath.com -gamingspark.com -ge0ip.com -genericviagrasildenafiled.net -geoads.com -get-free-social-traffic.com -gk170.ru -golfresa.lucania.se -gopixdatabase.com -gpms.org.my -gribkovye-zabolevaniya.com -gsasearchenginerankerhelp.com -guiadeserraazul.com +ezb.elvenmachine.com +f0815.de +face.hostingx.eu +fighrofacciufreesig.ga +forensicpsychiatry.ru +frbizlist.com +free-gluten.ru +free-share-buttons.blogspot.com +froggytube.com +funponsel.com +games.kolossale.ru +gearsadspromo.club +geckoandfly.com +get-seo-domain.com +getlaid-xxxhookupdirect.com +gloverid.site +goblacked.com +grtyi.com +gta-club.ru +gwagka.com hard-porn.mobi -heartofpayne.xyz -hermesbracelets.xyz -hmywwogw.bloger.index.hr -homedecorpicture.us -hotblognetwork.com -hotenergy.ru -imitex-plus.ru -internet-apteka.ru -jobgirl24.ru -jolic2.com -karting196.ru -ketrzyn.pl -kino-filmi.com -kmd-pto.ru -laexotic.com -lalalove.ru -lasvegaslockandsafe.com -law-check-eight.xyz -ledis.top -levaquin750mg.blogspot.com -liupis.com -localmatchbook.com -lomb.co -lowephotos.info -m292.info -manualterap.roleforum.ru -mararoom.ru +hdpixion.com +hentai-manga.porn +home-task.com +homedo.fabpage.com +hoverboard360.at +hoverboard360.nl +hspline.com +htmlcorner.com +infokonkurs.ru +interior-stickers.ru +iphantom.com +iplogger.org +isitpaleo.info +it-max.com.ua +jerseychinabizwholesale.com +jongose.ninja +jrcigars.com +kabbalah-red-bracelets.com +kerwinandcariza.com +kleine-titten.biz +klumba55.ru +konoplisemena.com +koopilka.com +kosova.de +leadwayau.com +letmacwork.world +likrot.com +littlecity.ch +livingroomdecor.info +locustdesign.co +mall.uk +mamasuper.prom.ua masthopehomes.com -medicineseasybuy.com -mistr-x.org -mmstat.com -moetomnenie.com -moroccosurfadventures.com -mostantikor.ru -mp3downloadhq.com +mediaoffers.click +medicines-choice.com +medtherapy.ru +megakino.net +middlerush-a.akamaihd.net +mobstarr.com +modnie-futbolki.net +moje-recenze.cz +most.gov.iq +motherboard.vice.com +mrcsa.com.au mrpornogratis.xxx -muz-tracker.net -my-bc.ru -mydownloadengine.com -nagdak.ru -narutonaruto.ru -nash-krym.info -nfljerseyscheapbiz.us -nuit-artisanale.com -nyfinance.ml -obiavo.net -olympescort.com -onemantrip.com -oneshotdate.com -onstrapon.purplesphere.in -ooo-gotovie.ru -outpersonals.com -pavlodar.xkaz.org -penisvergrotendepillennl.ovh -pensplan.com -perso.wanadoo.es -pflexads.com -phobia.us -photo.houseofgaga.ru -photokitchendesign.com -picsfair.com -pila.pl -pinsdaddy.com -plastjulgranar.se -poiskzakona.ru -porno-home365.com -pornofiljmi.com -pozdravleniya-c.ru -pozdrawleniya.ru +muzaporn.com +my-floor.in.ua +myhydros.org +naperehresti.info +narkologiya-peterburg.ru +neodownload.webcam +nl.netlog.com +okonich.com.ua +okroshki.ru +one.net.in +onlineporno.site +openlibrary.org +pc-virus-d0l92j2.pw +pertlocogasilk.tk +pinapchik.com +pio.polytopesexempt.com +pix-hd.com +popunder.ru +porn555.com +pornmania.pl +pornoreino.com +pornowarp.info praisong.net -prizesbook.online -promoheads.com -propranolol40mg.blogspot.com -pufip.com -purplesphere.in -quickcashlimited.com -rank-checker.online -ranksonic.info -rapidsites.pro -razorweb-a.akamaihd.net -remybutler.fr -rentaremotecomputer.com -replicaclub.ru -rn-to-bsn.com -rocis.site -rocketchange.ru -rockingclicks.com -rossanasaavedra.net -royalvegascasino.com -rubbed.us -rus-teh.narod.ru -rusexy.xyz +profit-opportunity.com +projectforte.ru +prom23.ru +promover.org +proprostatit.com +pshare.biz +putevka24.ru +qld10000.net +rabotaetvse.ru +rating-bestcasino.com +rcpmda.ikan1080.xyz +registry-clean-up.net +relax.ru +remont-ustanovka-tehniki.ru +rennlist.com +richinvestmonitor.com +rockma.se +royal-betting.net +royalcar-ufa.ru +ruscopybook.com +russia-today-video.ru +rutor.vip +s.lollypopgaming.com +saddiechoua.com +sc-specialhost.com scanner-elena.top -scanner-nelson.top -scmor.ilxc.cc -see-your-website-here.com -senger.atspace.co.uk -seoexperimenty.ru -shivafurnishings.com -shtaketniki.ru -sildenafil-tadalafil.info -sildenafilcitratemed.com -sladkoevideo.com -smartshoppymac.com -space2019.top -spb.ru -stair.registrydomainservices.com -startwp.org +score-ads.men +seoanalyses.com +sexystrippe.info +sharebutton.net +sharpchallenge.com +silktide.com +sim-service.net +sitevaluation.org +smartadserver.com +snjack.info +socialbutton.xyz +solitaire-game.ru +sotkal.lark.ru +spl63.fr +squidoo.com sterva.cc -stjamesschool.info -sugarlyflex.pw -suhanpacktech.com -swinon.site -t3chtonic.com -tako3.com -tamada69.com -testingads.pro -theallgirlarcade.com -thepantonpractice.co.uk -tiens2010.ru -tivolibasket.it -tizanidinenarcotic.blogspot.com -tizanidineoral.blogspot.com -tizanidinerestlesslegsyndrome.blogspot.com -top1-seo-service.com -topbestgames.com -topsy.com -track2.shop -tracklead.net -tracxn.com -traffique.net -tvand.ru -tvorozhnaja-zapekanka-recept.ru -twittrading.com -u555u.info -unrealcommander.org -vchulkax.com -video-hollywood.ru -videochat.mx -videogamesecrets.com -vigrx-original.ru -virtuagirl.com -vseigry.fun +streamin.to +superoboi.com.ua +talant-factory.ru +taqywu51.soup.io +techart24.com +theautoprofit.ml +tizanidinehydrochloride4mgstreetvalue.blogspot.com +tmm-kurs.ru +tour-line.net +tpu.ru +trahvid.com +trubywriting.com +trustl.life +u-cheats.ru +ultimatesetnewfreeallsoftupgradesystems.pw +uptime-gamma.net +vbtracker.net +vektorpress.ru +viagengrarx.com +viagraneggrx.com +viktoria-center.ru +vseigru.one vucms.com -weightatraining.com -welcomeauto.ru -whereverdesperate.gq -wholesalejerseyscheapjerseys.us.com -xboxster.ru -xkaz.org -xn----9sbebi2bvzr7h.xn--p1ai -xn--80aaafbn2bc2ahdfrfkln6l.xn--p1ai -xn--80aaaks3bbhabgbigamdr2h.xn--p1ai -yaoguangdj.com -zaimite.ru -zeroredirect7.com +w3data.co +warning.or.kr +webjam.com +website-speed-check.site +website-stealer.nufaq.com +wieseversa.no +windowssearch-exp.com +xn--90acenikpebbdd4f6d.xn--p1ai +xn--e1afanlbnfckd7c3d.xn--p1ai +xn--oogle-wmc.com +xtube.com +xxxguitars.com +yatrk.xyz +zdesformula.ru +zeikopay.com +zhacker.net +zhongwenlink.com +zoodrawings.com +zoompegs.com +zryydi.com +zverokruh-shop.cz diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index a229fc66b..687d89316 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.227 0; +10.1.0.76 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index 717f7d2c5..941f1689c 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -44904,3 +44904,1251 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2487 +### Updated: Wed Jul 28 07:39:59 UTC 2021 +### Bad Bot Count: 615 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ALittle Client +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ + diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 717f7d2c5..941f1689c 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -44904,3 +44904,1251 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2487 +### Updated: Wed Jul 28 07:39:59 UTC 2021 +### Bad Bot Count: 615 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ALittle Client +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ +