diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index a0a7e8cb0..8085c0a67 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2488 +#### Version: V4.2021.07.2489 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [616](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 595ac6493..abeacd106 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2488 +#### Version: V4.2021.07.2489 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [616](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index 6702a6ab8..e2ea36367 100755 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2021.07.2488 +#### Version: V4.2021.07.2489 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [616](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list index 98997b8ee..8c9da7c4a 100755 --- a/_generator_lists/bad-user-agents.list +++ b/_generator_lists/bad-user-agents.list @@ -1,4 +1,3 @@ -ubermetrics-technologies.com 01h4x.com 360Spider 404checker @@ -607,6 +606,7 @@ sysscan tAkeOut trendiction.com trendiction.de +ubermetrics-technologies.com voyagerx.com webmeup-crawler x09Mozilla diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 14828ff9b..6b329923b 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2021.07.2488 +#### Version: V4.2021.07.2489 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [616](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 0d1ceb108..f97b2e17d 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2488 -### Updated: Wed Jul 28 14:07:30 UTC 2021 +### Version: V4.2021.07.2489 +### Updated: Thu Jul 29 11:09:36 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 615 +### Bad Bot Count: 616 ################################################### ### VERSION INFORMATION ## @@ -755,6 +755,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)tAkeOut(?:\b)" 3; "~*(?:\b)trendiction.com(?:\b)" 3; "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)ubermetrics-technologies.com(?:\b)" 3; "~*(?:\b)voyagerx.com(?:\b)" 3; "~*(?:\b)webmeup-crawler(?:\b)" 3; "~*(?:\b)x09Mozilla(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index e2e5a5791..888ce68f2 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2488 -### Updated: Wed Jul 28 14:07:31 UTC 2021 +### Version: V3.2021.07.2489 +### Updated: Thu Jul 29 11:09:37 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 615 +### Bad Bot Count: 616 ################################################### ### VERSION INFORMATION ## @@ -755,6 +755,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)tAkeOut(?:\b)" 3; "~*(?:\b)trendiction.com(?:\b)" 3; "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)ubermetrics-technologies.com(?:\b)" 3; "~*(?:\b)voyagerx.com(?:\b)" 3; "~*(?:\b)webmeup-crawler(?:\b)" 3; "~*(?:\b)x09Mozilla(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 0d1ceb108..f97b2e17d 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2488 -### Updated: Wed Jul 28 14:07:30 UTC 2021 +### Version: V4.2021.07.2489 +### Updated: Thu Jul 29 11:09:36 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 615 +### Bad Bot Count: 616 ################################################### ### VERSION INFORMATION ## @@ -755,6 +755,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)tAkeOut(?:\b)" 3; "~*(?:\b)trendiction.com(?:\b)" 3; "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)ubermetrics-technologies.com(?:\b)" 3; "~*(?:\b)voyagerx.com(?:\b)" 3; "~*(?:\b)webmeup-crawler(?:\b)" 3; "~*(?:\b)x09Mozilla(?:\b)" 3; diff --git a/dev-tools/_robots_input/robots-input.txt b/dev-tools/_robots_input/robots-input.txt index 2f884078e..6adaaaad2 100644 --- a/dev-tools/_robots_input/robots-input.txt +++ b/dev-tools/_robots_input/robots-input.txt @@ -606,6 +606,7 @@ sysscan tAkeOut trendiction.com trendiction.de +ubermetrics-technologies.com voyagerx.com webmeup-crawler x09Mozilla diff --git a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf index e2e5a5791..888ce68f2 100644 --- a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2488 -### Updated: Wed Jul 28 14:07:31 UTC 2021 +### Version: V3.2021.07.2489 +### Updated: Thu Jul 29 11:09:37 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 615 +### Bad Bot Count: 616 ################################################### ### VERSION INFORMATION ## @@ -755,6 +755,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)tAkeOut(?:\b)" 3; "~*(?:\b)trendiction.com(?:\b)" 3; "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)ubermetrics-technologies.com(?:\b)" 3; "~*(?:\b)voyagerx.com(?:\b)" 3; "~*(?:\b)webmeup-crawler(?:\b)" 3; "~*(?:\b)x09Mozilla(?:\b)" 3; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 0f898d924..de65f07dd 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.217 1; +10.1.1.31 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index e0a7c466d..92ab0df4c 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.217 0; +10.1.1.31 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index e2e5a5791..888ce68f2 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2488 -### Updated: Wed Jul 28 14:07:31 UTC 2021 +### Version: V3.2021.07.2489 +### Updated: Thu Jul 29 11:09:37 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 615 +### Bad Bot Count: 616 ################################################### ### VERSION INFORMATION ## @@ -755,6 +755,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)tAkeOut(?:\b)" 3; "~*(?:\b)trendiction.com(?:\b)" 3; "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)ubermetrics-technologies.com(?:\b)" 3; "~*(?:\b)voyagerx.com(?:\b)" 3; "~*(?:\b)webmeup-crawler(?:\b)" 3; "~*(?:\b)x09Mozilla(?:\b)" 3; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 0f898d924..de65f07dd 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.217 1; +10.1.1.31 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index e0a7c466d..92ab0df4c 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.217 0; +10.1.1.31 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index e2e5a5791..888ce68f2 100644 --- a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2488 -### Updated: Wed Jul 28 14:07:31 UTC 2021 +### Version: V3.2021.07.2489 +### Updated: Thu Jul 29 11:09:37 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 615 +### Bad Bot Count: 616 ################################################### ### VERSION INFORMATION ## @@ -755,6 +755,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)tAkeOut(?:\b)" 3; "~*(?:\b)trendiction.com(?:\b)" 3; "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)ubermetrics-technologies.com(?:\b)" 3; "~*(?:\b)voyagerx.com(?:\b)" 3; "~*(?:\b)webmeup-crawler(?:\b)" 3; "~*(?:\b)x09Mozilla(?:\b)" 3; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 0f898d924..de65f07dd 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.217 1; +10.1.1.31 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf index d4d448d85..1cee28ac6 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -667,6 +667,7 @@ "~*(?:\b)tAkeOut(?:\b|)" 0; "~*(?:\b)trendiction.com(?:\b|)" 0; "~*(?:\b)trendiction.de(?:\b|)" 0; + "~*(?:\b)ubermetrics-technologies.com(?:\b|)" 0; "~*(?:\b)voyagerx.com(?:\b|)" 0; "~*(?:\b)webmeup-crawler(?:\b|)" 0; "~*(?:\b)x09Mozilla(?:\b|)" 0; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index e0a7c466d..92ab0df4c 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.217 0; +10.1.1.31 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index e2e5a5791..888ce68f2 100644 --- a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2488 -### Updated: Wed Jul 28 14:07:31 UTC 2021 +### Version: V3.2021.07.2489 +### Updated: Thu Jul 29 11:09:37 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 615 +### Bad Bot Count: 616 ################################################### ### VERSION INFORMATION ## @@ -755,6 +755,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)tAkeOut(?:\b)" 3; "~*(?:\b)trendiction.com(?:\b)" 3; "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)ubermetrics-technologies.com(?:\b)" 3; "~*(?:\b)voyagerx.com(?:\b)" 3; "~*(?:\b)webmeup-crawler(?:\b)" 3; "~*(?:\b)x09Mozilla(?:\b)" 3; diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index 38e2443e7..697ade4bd 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -2488 +2489 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index e2e5a5791..888ce68f2 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2488 -### Updated: Wed Jul 28 14:07:31 UTC 2021 +### Version: V3.2021.07.2489 +### Updated: Thu Jul 29 11:09:37 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 615 +### Bad Bot Count: 616 ################################################### ### VERSION INFORMATION ## @@ -755,6 +755,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)tAkeOut(?:\b)" 3; "~*(?:\b)trendiction.com(?:\b)" 3; "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)ubermetrics-technologies.com(?:\b)" 3; "~*(?:\b)voyagerx.com(?:\b)" 3; "~*(?:\b)webmeup-crawler(?:\b)" 3; "~*(?:\b)x09Mozilla(?:\b)" 3; diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index 0d1ceb108..f97b2e17d 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2488 -### Updated: Wed Jul 28 14:07:30 UTC 2021 +### Version: V4.2021.07.2489 +### Updated: Thu Jul 29 11:09:36 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 615 +### Bad Bot Count: 616 ################################################### ### VERSION INFORMATION ## @@ -755,6 +755,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)tAkeOut(?:\b)" 3; "~*(?:\b)trendiction.com(?:\b)" 3; "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)ubermetrics-technologies.com(?:\b)" 3; "~*(?:\b)voyagerx.com(?:\b)" 3; "~*(?:\b)webmeup-crawler(?:\b)" 3; "~*(?:\b)x09Mozilla(?:\b)" 3; diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index a1c85c7df..6a2d88156 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -13.66.169.42 0; +52.170.156.225 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 0d1ceb108..f97b2e17d 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2488 -### Updated: Wed Jul 28 14:07:30 UTC 2021 +### Version: V4.2021.07.2489 +### Updated: Thu Jul 29 11:09:36 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 615 +### Bad Bot Count: 616 ################################################### ### VERSION INFORMATION ## @@ -755,6 +755,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)tAkeOut(?:\b)" 3; "~*(?:\b)trendiction.com(?:\b)" 3; "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)ubermetrics-technologies.com(?:\b)" 3; "~*(?:\b)voyagerx.com(?:\b)" 3; "~*(?:\b)webmeup-crawler(?:\b)" 3; "~*(?:\b)x09Mozilla(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 0f898d924..de65f07dd 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.217 1; +10.1.1.31 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index e0a7c466d..92ab0df4c 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.217 0; +10.1.1.31 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 0d1ceb108..f97b2e17d 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2488 -### Updated: Wed Jul 28 14:07:30 UTC 2021 +### Version: V4.2021.07.2489 +### Updated: Thu Jul 29 11:09:36 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 615 +### Bad Bot Count: 616 ################################################### ### VERSION INFORMATION ## @@ -755,6 +755,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)tAkeOut(?:\b)" 3; "~*(?:\b)trendiction.com(?:\b)" 3; "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)ubermetrics-technologies.com(?:\b)" 3; "~*(?:\b)voyagerx.com(?:\b)" 3; "~*(?:\b)webmeup-crawler(?:\b)" 3; "~*(?:\b)x09Mozilla(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 0f898d924..de65f07dd 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.217 1; +10.1.1.31 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index e0a7c466d..92ab0df4c 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.217 0; +10.1.1.31 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 0d1ceb108..f97b2e17d 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2488 -### Updated: Wed Jul 28 14:07:30 UTC 2021 +### Version: V4.2021.07.2489 +### Updated: Thu Jul 29 11:09:36 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 615 +### Bad Bot Count: 616 ################################################### ### VERSION INFORMATION ## @@ -755,6 +755,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)tAkeOut(?:\b)" 3; "~*(?:\b)trendiction.com(?:\b)" 3; "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)ubermetrics-technologies.com(?:\b)" 3; "~*(?:\b)voyagerx.com(?:\b)" 3; "~*(?:\b)webmeup-crawler(?:\b)" 3; "~*(?:\b)x09Mozilla(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 0f898d924..de65f07dd 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.217 1; +10.1.1.31 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index d4d448d85..1cee28ac6 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -667,6 +667,7 @@ "~*(?:\b)tAkeOut(?:\b|)" 0; "~*(?:\b)trendiction.com(?:\b|)" 0; "~*(?:\b)trendiction.de(?:\b|)" 0; + "~*(?:\b)ubermetrics-technologies.com(?:\b|)" 0; "~*(?:\b)voyagerx.com(?:\b|)" 0; "~*(?:\b)webmeup-crawler(?:\b|)" 0; "~*(?:\b)x09Mozilla(?:\b|)" 0; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index e0a7c466d..92ab0df4c 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.217 0; +10.1.1.31 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 0d1ceb108..f97b2e17d 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2488 -### Updated: Wed Jul 28 14:07:30 UTC 2021 +### Version: V4.2021.07.2489 +### Updated: Thu Jul 29 11:09:36 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 615 +### Bad Bot Count: 616 ################################################### ### VERSION INFORMATION ## @@ -755,6 +755,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)tAkeOut(?:\b)" 3; "~*(?:\b)trendiction.com(?:\b)" 3; "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)ubermetrics-technologies.com(?:\b)" 3; "~*(?:\b)voyagerx.com(?:\b)" 3; "~*(?:\b)webmeup-crawler(?:\b)" 3; "~*(?:\b)x09Mozilla(?:\b)" 3; diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index a1c85c7df..6a2d88156 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -13.66.169.42 0; +52.170.156.225 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 00f66f09a..9ee9d9ecb 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2488 -### Updated: Wed Jul 28 14:07:30 UTC 2021 +### Version: V4.2021.07.2489 +### Updated: Thu Jul 29 11:09:36 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 615 +### Bad Bot Count: 616 ################################################### ### VERSION INFORMATION ## @@ -755,6 +755,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)tAkeOut(?:\b)" 3; "~*(?:\b)trendiction.com(?:\b)" 3; "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)ubermetrics-technologies.com(?:\b)" 3; "~*(?:\b)voyagerx.com(?:\b)" 3; "~*(?:\b)webmeup-crawler(?:\b)" 3; "~*(?:\b)x09Mozilla(?:\b)" 3; diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index a1c85c7df..6a2d88156 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -13.66.169.42 0; +52.170.156.225 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 5add926e6..5dd0ca518 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2488 -### Updated: Wed Jul 28 14:07:30 UTC 2021 +### Version: V4.2021.07.2489 +### Updated: Thu Jul 29 11:09:36 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 615 +### Bad Bot Count: 616 ################################################### ### VERSION INFORMATION ## @@ -755,6 +755,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)tAkeOut(?:\b)" 3; "~*(?:\b)trendiction.com(?:\b)" 3; "~*(?:\b)trendiction.de(?:\b)" 3; + "~*(?:\b)ubermetrics-technologies.com(?:\b)" 3; "~*(?:\b)voyagerx.com(?:\b)" 3; "~*(?:\b)webmeup-crawler(?:\b)" 3; "~*(?:\b)x09Mozilla(?:\b)" 3; diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index 0f898d924..de65f07dd 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.217 1; +10.1.1.31 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/blacklist-user-agents.conf b/dev-tools/test_units/blacklist-user-agents.conf index d4d448d85..1cee28ac6 100644 --- a/dev-tools/test_units/blacklist-user-agents.conf +++ b/dev-tools/test_units/blacklist-user-agents.conf @@ -667,6 +667,7 @@ "~*(?:\b)tAkeOut(?:\b|)" 0; "~*(?:\b)trendiction.com(?:\b|)" 0; "~*(?:\b)trendiction.de(?:\b|)" 0; + "~*(?:\b)ubermetrics-technologies.com(?:\b|)" 0; "~*(?:\b)voyagerx.com(?:\b|)" 0; "~*(?:\b)webmeup-crawler(?:\b|)" 0; "~*(?:\b)x09Mozilla(?:\b|)" 0; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 63385ef8d..752a4dbbf 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -404enemy -80legs -ExtLinksBot -ExtractorPro +CensysInspect +Demon +Id-search +IndeedBot NetMechanic -PyCurl -RankingBot2 SiteCheckerBotCrawler -Sqlworm -WebWhacker +Web Collage +muhstik-scan +sexsearcher +x22Mozilla diff --git a/dev-tools/test_units/random-bots-for-test.list b/dev-tools/test_units/random-bots-for-test.list index 3407a98e8..c7b0e1c34 100644 --- a/dev-tools/test_units/random-bots-for-test.list +++ b/dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -AIBOT -Aboundexbot +01h4x.com AiHitBot +Aipbot +Alexibot +AllSubmitter +Anarchy Anarchy99 Ankit Apexoo Aspiegel -Asterias Attach +AwarioSmartBot +BDFetch BLEXBot -BackStreet -BackWeb -Backlink-Ceck -BacklinkCrawler -Bandit +Barkrowler +BatchFTP Battleztar Bazinga Bigfoot -Blackboard +Bitacle +Black Hole Blow -Bolt -Buddy +BuiltBotTough BuiltWith -CCBot +Bullseye +BuzzSumo CODE87 -Calculon -Cegbfeieh +CSHttp +CheTeam CheeseBot -CherryPicker -ChinaClaw +Chlooe +Claritybot Cloud mapping -Cocolyzebot Copier -CopyRightCheck -Craftbot -Crescent +Cosmos +CrazyWebCrawler CrunchBot Curious +Custo CyotekWebCopy +DIIbot DTS Agent -Devil +DatabaseDriverMysqli +Demon Digincore -DigitalPebble -Dirbuster -Disco Dispatch DnyzBot DomCopBot -DomainCrawler +DomainAppender +DomainSigmaCrawler +DomainStatsBot Domains Project Dotbot -Dragonfly Drip ECCP/1.0 +EMail Wolf +EasyDL +Ecxi +EirGrabber +EroCrawler Evil +Exabot +Express WebPictures Extractor +Extreme Picture Finder EyeNetIE Ezooms -FHscan -Fimap -FlashGet -Flunky Freeuploader -Fuzz FyberSpider Fyrebot -G-i-g-a-b-o-t GT::WWW GalaxyBot +GermCrawler +GetRight +GetWeb Getintent -Go!Zilla -GoZilla +Gotit GrabNet -GridBot -HEADMasterSEO +Grafula +GrapeFX +HMView HTMLparser -Harvest +HTTrack +HaosouSpider +Heritrix +Hloader Humanlinks HybridBot -IDBot -IRLbot +IDBTE4M Id-search +Image Fetch Image Sucker -IndeedBot -InfoNaviRobot -InterGET -Internet Ninja -Iria +InternetSeer Iskanie JOC Web Spider -Jbrofuzz -Jetty -JikeSpider +JennyBot +JustView Jyxobot Kenjin Spider Kinza Kozmosbot -LNSpiderguy -Larbin +Lanshanbot LeechFTP +LeechGet +Lftp LibWeb -LinkWalker +LinkextractorPro LinkpadBot +LinksManager +LinqiaMetadataDownloaderBot LinqiaScrapeBot +Lipperhey Lipperhey Spider -Ltx71 -MFC_Tear_Sample +Litemage_walker +MIDown tool MIIxpc MJ12bot MQQBrowser -MSFrontPage -MSIECrawler -Magnet +MTRobot Majestic SEO Majestic-SEO -Majestic12 -Masscan -Mata Hari -MeanPath Bot -MegaIndex.ru -Mister PiX +MauiBot +Mb2345Browser +Mediatoolkitbot +MicroMessenger +Moblie Safari +Mojeek +Mojolicious NICErsPRO -NPbot -Needle -NetSpider -NetZIP -Netcraft -Nettrack -Netvibes +Navroad +Nessus +Net Vampire +NetAnts +NetLyzer +NetMechanic +NextGenSearchBot Nibbler -Nikto -NimbleCrawler -Nimbostratus -Ninja +Niki-bot +Nuclei Nutch -Offline Explorer -OnCrawl +OpenLinkProfiler Openfind -OrangeSpider -OutfoxBot -PHPCrawl -POE-Component-Client-HTTP -Page Analyzer +Openvas +OrangeBot +PECL::HTTP PageAnalyzer -PageScorer +PageThing.com Pandalytics -Panscient PeoplePal +Petalbot +Picscout Picsearch -Pimonster -Pixray -PleaseCrawl +PictureFinder +Pockey ProWebWalker +Probethenet Psbot +Pu_iN PxBroker -QueryN Metasearch -RankActive +Quick-Crawler +RSSingBot RankFlex +RankingBot2 Re-re ReGet -Reaper +RealDownload RebelMouse +Recorder +SBIder SMTBot -ScanAlert -ScoutJet +SalesIntelligent +Scrapy Screaming -ScrepyBot -SemrushBot -SeobilityBot +ScreenerBot +SearchmetricsBot Seomoz -Siphon +Shodan SiteExplorer -SiteSnagger -Siteimprove +SiteLockSpider +Sitebeam +SlySearch Snapbot -Snoopy -Sottopop -Spammen +Sociscraper +SpaceBison Spanner -Spbot -Spinn3r -Stripper -SuperBot +SputnikBot +Sqworm +Steeler Suzuran +Swiftbot Szukacz -T8Abot -Telesoft -Telesphorep +TeleportPro The Intraformant -Toata +TightTwatBot +Toweyabot Tracemyfile Trendictionbot True_Robot -URLy Warning -URLy.Warning -Vagabondo -VidibleScraper -VoidEYE -Voil +TwengaBot +Upflow +Vacuum +VeriCiteCrawler WBSearchBot -WEBDAV -WISENutbot WPScan +Web Auto Web Collage -Web Fetch -Web Fuck +Web Pix Web Sauger -WebAuto -WebBandit -WebFetch -WebGo IS +Web Sucker +WebCollage +WebFuck WebPix -WebSauger -WebStripper -WebSucker Webalta Webshag Website Quester -Webster +WebsiteExtractor Whacker Who.is Bot WinHTTrack +WiseGuys Robot Wonderbot Woobot -Wotbox -Xaldon_WebSpider +Xenu YoudaoBot -Zauba -Zeus -Zitebot -ZmEu -ZoominfoBot +Zade arquivo-web-crawler -com.plumanalytics -domainsproject.org -eCatch +autoemailspider +backlink-check +check1.exe +coccocbot-web +cognitiveseo +evc-batch +gopher heritrix -magpie-crawler -meanpathbot +instabid +ips-agent +lwp-trivial +netEstate NE Crawler +page scorer plumanalytics -ripz -satoristudio.net scan.lol -seobility -seoscanners serpstatbot sexsearcher -sogouspider -trendiction.com +sitechecker.pro +sp_auditbot +spyfu +sysscan +tAkeOut +ubermetrics-technologies.com webmeup-crawler -zauba.io +zgrab diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 2aa06a5d7..5da5c87fe 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -360Spider -ALittle Client -ASPSeek +404checker +404enemy Abonti Aboundex -AfD-Verbotsverfahren AhrefsBot -Aipbot Alexibot AllSubmitter -AlphaBot -Anarchy99 +Anarchie +Apexoo +Aspiegel AwarioSmartBot -BDCbot -BackDoorBot -BacklinkCrawler -Bandit -BatchFTP +BackStreet +Barkrowler Battleztar Bazinga BetaBot -Bigfoot Bitacle -Blow -Brandwatch +Blackboard +BlowFish +Boardreader +Bolt +Buck +Bullseye BuzzSumo -Calculon CazoodleBot -Cegbfeieh -ChinaClaw -Chlooe -Claritybot +CensysInspect +CheeseBot +CherryPicker +Cliqzbot +Cogentbot +Collector Copyscape -Cosmos -Craftbot -CrazyWebCrawler Custo -DBLBot -DSearch -DTS Agent -Demon -Devil +DIIbot +DataCha0s +DatabaseDriverMysqli Dirbuster -Disco Discobot Discoverybot -Dispatch DittoSpyder -DomCopBot -DomainAppender -Domains Project -Dragonfly +DomainCrawler +DomainStatsBot Drip -EMail Wolf -EasyDL -Ebingbong -EirGrabber -Evil Express WebPictures +ExtLinksBot ExtractorPro +Extreme Picture Finder EyeNetIE +FHscan FemtosearchBot -Fimap +Firefox/7.0 FlashGet -Flunky -Foobot -Fuzz FyberSpider +Fyrebot G-i-g-a-b-o-t -GT::WWW -Genieo +GermCrawler GetRight -GetWeb -Getintent Gigabot Go!Zilla -Gotit -GrapeshotCrawler -GridBot +Grabber +Grafula +GrapeFX HEADMasterSEO -HMView +HTMLparser HTTP::Lite HTTrack +Haansoft Havij -Hloader -InfoNaviRobot +Heritrix +HybridBot +IDBTE4M +IRLbot +Id-search +IlseBot +InfoTekies +Intelliseek InterGET InternetSeer -Iria IstellaBot JetCar +Jetty JikeSpider -Joomla Keyword Density -Kinza +Kozmosbot +LNSpiderguy Lanshanbot -LeechFTP +Larbin LeechGet -Lftp -LibWeb -Libwhisker LieBaoFast Lightspeedsystems +Likse LinkScan -LinkWalker -LinksManager +LinkpadBot LinqiaMetadataDownloaderBot -Lipperhey Spider -MIDown tool +LinqiaScrapeBot +Litemage_walker MIIxpc -MQQBrowser +MSIECrawler Magnet Mail.RU_Bot -Majestic12 +Majestic-SEO MarkWatch -Masscan +Mass Downloader Mata Hari +MauiBot MeanPath Bot Meanpathbot Mediatoolkitbot -Microsoft URL Control -Moblie Safari -Morfeus Fucking Scanner -Mozlila -Mr.4x3 -NPbot +MegaIndex.ru +Msrabot +Musobot Name Intelligence Nameprotect -Navroad -Nessus Net Vampire -Nibbler +NetAnts +NetZIP +Netcraft Niki-bot Nikto +Nmap Nuclei -Octopus +Nutch +Offline Explorer Offline Navigator -OpenLinkProfiler OpenVAS -POE-Component-Client-HTTP +Openvas +OrangeSpider +OutclicksBot +PECL::HTTP +PHPCrawl PageThing.com +Panscient Papa Foto Petalbot +Picscout +Picsearch +PictureFinder +Pimonster +Pixray PleaseCrawl -Pockey -Psbot +ProPowerBot +ProWebWalker +Pump PxBroker -RSSingBot -RankFlex -RankingBot2 +Quick-Crawler +RankActiveLinkBot Rankivabot RankurBot -ReGet -RebelMouse +Re-re +RealDownload +Reaper +RedesScrapy RepoMonkey RocketCrawler -Rogerbot -SEOkicks-Robot -SEOprofiler +SEOlyticsCrawler +SEOstats SMTBot -ScanAlert -ScoutJet -ScreenerBot -ScrepyBot -Searchestate +Scanbot +Scrapy SearchmetricsBot Siphon -Site Sucker +SiteExplorer +SiteLockSpider Sitebeam -SlySearch -Snake -Snapbot -Sogou web spider +Siteimprove +SmartDownload +Snoopy +SocialRankIOBot +Sociscraper +Sosospider +Sottopop +SpaceBison Spammen -Spinn3r +SputnikBot Sqlworm -Stripper +Sqworm +Steeler +Sucker Sucuri -SuperBot -Teleport +SuperHTTP +Suzuran Telesoft Telesphorep -Thumbor -Titan Toata Tracemyfile Trendiction -True_Robot -Turingos -Turnitin TurnitinBot -TwengaBot -URLy Warning -V-BOT -Vacuum -Voil -WBSearchBot +Upflow +VelenPublicWebCrawler +Virusdie +WASALive-Bot WEBDAV WISENutbot +WWW-Mechanize WWW::Mechanize WWWOFFLE Wallpapers/3.0 +WallpapersHD WeSEE -Web Auto Web Collage -Web Enhancer +Web Fetch +Web Fuck Web Sauger -WebEnhancer -WebFetch +WebAuto +WebGo IS +WebLeacher WebPix -WebStripper +WebSauger WebWhacker -Webalta -WebmasterWorldForumBot -WebsiteExtractor -Webster +Website Quester Whack +Whacker Whatweb +Who.is Bot Widow -WinHTTrack -WiseGuys Robot Wonderbot -Xaldon WebSpider +Wotbox +Wprecon +Xaldon_WebSpider Zauba -Zermelo -ZmEu -ZoomBot -ZumBot ZyBorg arquivo-web-crawler -com.plumanalytics -crawl.sogou.com +arquivo.pt +autoemailspider +backlink-check +check1.exe +crawler.feedback crawler4j -evc-batch -instabid +demandbase-bot +facebookscraper +heritrix ips-agent +isitwp.com +lwp-request lwp-trivial -mediawords -muhstik-scan +meanpathbot +oBot page scorer +pcBrowser +polaris version +probe-image-size s1z.ru -satoristudio.net -seostar +scan.lol +seobility sexsearcher +sitechecker.pro siteripz -spyfu -sysscan -trendiction.de +sogouspider +tAkeOut xpymep1.exe zauba.io -zgrab diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 47f32e2ec..664fc44ac 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -agreda.pluto.ro -allsilver925.co.il -babyguns.ru -conocer-sanabria.com -gojiberriess.apishops.ru -ideawheel.com -lulea-auktionsverk.se -menhealed.net -mere.host.sk -youresponsive.com +ahrntal.verymes.xyz +angel1777.kz +dosugrostov.site +everypony.ru +fuckingawesome.com +new-apps.ru +npoet.ru +shopperifymac.com +siteaero.com +soheavyblog.com diff --git a/dev-tools/test_units/random-referrers-for-test.list b/dev-tools/test_units/random-referrers-for-test.list index 8a99612ca..2a55fc719 100644 --- a/dev-tools/test_units/random-referrers-for-test.list +++ b/dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -00go.com -11pikachu.ru -178evakuator178.ru -4kplayer.pl -8xv8.com -a96527gi.beget.tech -aa9bd78f328a6a41279d0fad0a88df1901.com -aa9d046aab36af4ff182f097f840430d51.com -absolute-s.ru -adexprts.com -adservme.com -adtiger.tk -adultnet.in -advancedmassagebysara.com -advancedsoftwaresupport.com -afslankpillen2017nl.eu +0lovespells0.blogspot.com +12masterov.com +1314dh.com +14b.info +2015god.org +256bit.by +2ads.co.uk +55wmz.ru +5elementov.ru +72-news.com +8kisses.eu +9icmzvn6.website +a14download.com +aa8780bb28a1de4eb5bff33c28a218a930.com +aanapa.ru +abusora.com +adanih.com +adclickservice.com +adnotbad.com +ads-cool.pro +ads.gold +adzerg.com +aerogo.com aktobe.xkaz.org -akuhni.by -alekseevec.ru -alfa-img.com -allknow.info -allnews24.in -allpdfmags.net -alvtank.se -amateurgalls.com -amateurmatch.com -amazon-adsystem.com -amoi.tn -animenime.ru -apibrowsesmartne-a.akamaihd.net -apifasterlightin-a.akamaihd.net -apimountainbikei-a.akamaihd.net -apple.com-webbrowsing-security.review -apple.com-webbrowsing-security.science -appmsr.org -arabseyes.com -ausmepa.org.au +alarmobninsk.ru +alborzan.com +albumsuper.info +alert-hgd.xyz +alfa9.com +alienwheels.de +aliexpress.com +alkoravto.ru +all4wap.ru +allnews.md +allstatesugarbowl.org +ameblo.jp +ananumous.ru +anapa-inns.ru +angkortours.vn +animalia-life.club +anjalika.co.in +apiitsthirteende-a.akamaihd.net +apispringsmartne-a.akamaihd.net +appartement-stumm.at +arendatora.ru +arkartex.ru +arquivo.pt +artel116.ru +asiengirls.net +asociatia-tipografilor-transilvania.ro +atmagroup.ru +atofilms.com +auto4style.ru +autonew.biz autoplate.info -autoseo-traffic.com -avtochehli.by -avtocredit-legko.ru -avtointeres.ru ayerbo.xhost.ro -azartclub.org -azarttoto.com -battle.net -battleforupdating.site -bavsac.com -bdsmgalls.net -bestdooz.com +babyfactory.fr +backlinkwatch.com +baditri.com +bagcionderlab.com +bangla-chat-uk-paralud.ga +barbourjackets.online +bard-real.com.ua +begalka.xyz +beget.tech +belinvestdom.by +bestofferswalkmydogouteveryday.gq +bestwaystogetpaid.us bestwebsiteawards.com -betshuckclean.com -beyan.host.sk -bfz.biz -bidbuy.co.kr -big-cash.net -bimmerpost.com -bingo8888.com -bird1.ru -biznesluxe.ru -blue-square.biz -bodybuilding-shop.biz -bolegapakistan.com -boltalko.xyz -bongiornos.info -bonkers.name -bonzibuddi.com -boole.onlinewebshop.net -bracketsmackdown.com -brainboostingsupplements.org -braintobe.win -brewdom.ru -brk-rti.ru -brokergid.ru -builtwith.com -buqayy0.livejournal.com +big-boards.info +bigpictures.club +blackcurranthumidifiers.site +blackle.com +blogporn.in +bluerobot.info +bonux.nextview.ru +bookmaker-bet.com +bookmark4you.com +bouda.kvalitne.cz +brandbucket.com +brendbutik.ru +brothers-smaller.ru +btnativenav.com +bumskontakte.org +buttons-for-your-website.com +buyclomidonlaine.com buyhoverboard.com -buynorxx.com -buypillsorderonline.com -buyscabiescream.com -buzzsumo.com -bwlx.prepedu.cn -cartierlovebracelet.xyz -casinopinup-wins.com -casinorewards.info -cdncash.net -cercacamion.it -cheap-trusted-backlinks.com -cheapbarbour.online -cheapcigarettesc.info -checkm8.com -chee-by.biz -chinaelitecheapjerseys.com -cineacademy.ru -cjmc.info -clicksotrk.com -club-musics.ru -cmd.kz -compliance-andrew.top -compliance-jessica.top -compliance-julia.top -compliance-willy.top -concordexoticrentals.com -coolingoods.com -coolyarddecorations.com -cordstrap.cc -cs-passion.pl -customergrowthsystems.com +buzzzg.men +cakemediahq.com.au +calc-for-credit.ru +camdolls.com +carloans.com +cartierwatch.xyz +cbb1smartlist12.click +cete.ru +chatroulette.world +cheapjerseysfootballshop.com +cherubinimobili.it +ci.ua +cinemaenergy-hd.ru +citizenclsdriveri7.pen.io +clash-clans.ru +click2pawn.com +clippingphotoindia.com +cocolyze.com +comissionka.net +compliance-fred.top +confib.ifmo.ru +contentsexpress.com +cookie-law-enforcement-ff.xyz +coolgramgoods.com +copenergo.ru +countbertwistdisp26.soup.io +country-chic.ru +crackguru.tk +crazyprotocol.com +creativehutindia.com +crest3d.ru +custodieva.ru +customchocolate.business-for-home.com +customcollegeessays.net czat.wp.pl -dailyfinancefix.com -danceuniverse.ru -daptravel.com -darodar.com -dating-time-now.com -dayibiao.com -ddlmega.net -de.zapmeta.com -dealwifi.com -degerlund.net -delitime.info -dichvuvesinhhanoi.com -dieswaene.com -dikqlyremy.info -dinkolove.ya.ru -docs4all.com -domoysshop.ru -dosugrostov.site -download-walpaperhd.blogspot.com -downloadkakaotalk.com -downloadme.life -dpihatinh.gov.vn -draniki.org -drillsaw.ru -drunkenstepfather.com -druzhininevgeniy63.blogspot.com -dtnlyss.com -dvr.biz.ua -e-biznes.info -eachdayisagift.review -ebooknovel.club -ecommerce-seo.org -edelstahlschornstein-123.de +dafatiri.com +dangphoto.trade +datadepths.com +deanmoore.ie +decorationspcs.com +defenderxtactical.com +derevesendeco.com +dev.citetick.com +doeco.ru +dolohen.com +domaincheck.io +domaincrawler.com +dotmass.top +doublepimp.com +download-of-the-warez.blogspot.com +drupa.com +dustyorate.com +ed-shop01.ru editmedios.com -editors.choice6912650.hulfingtonpost.com -ekto.ee -electricwheelchairsarea.com -elektrischezi.canalblog.com -elektrischezigarette1.blog.pl -elkacentr.ru -elstal.com.pl -empathica.com -enbersoft.com -enews.tech -ertelecom.ru -eshop.md +edudocs.net +elektrischezigaretten2.beeplog.com +elektronischezi.livejournal.com +elektrozigaretten1.webs.com +eloconcream.blogspot.com +emerson-rus.ru +en.home-task.com +englishdictionaryfree.com +enthuse.computernetworksonline.com +eorogo.top +erotikchat-24.com +es5.com +evehemming.blogspot.com.au exchangeit.gq -extstat.com -family1st.ca -fealq.com +exdocsfiles.com +expertblog.info +eyelike.com.ua +ez8motelseaworldsandiego.com +ezigarettenshop1.yolasite.com +f07.de +falcon-images.blogspot.com +farmingworm.com +feel-planet.com fefo.gdn -fickenbumsen.net -fifa-coins.online -fil.ru -filesvine.com -firma-legion.ru -firstsiteguide.com -fiverr.com -flashahead.info -footbalive.org -forensicpsychiatry.ru -forexgb.ru -formulaf1results.blogspot.com -forpackningsutveckling.se -fortwosmartcar.pw -forzeronly.com -free-today.com -frighteningremain.cf -funcrushgames.com -fungamelands.com -funnypica.com -gallerily.com -gamedayhouse.com -gardene.ru -gcup.ru -gerhardhealer.com -getlaid-xxxhookupdirect.com -gifspics.com -gk-atlant.info -glall.ru -goforexvps.com -goodly.pro -goodprotein.ru -gosarhivrt.ru -gotomontenegro.net -gratuitxblcodes.com -grizzlysgrill.com -growmyfunds.ca -gsmtlf.ru +felizporno.com +fetroshok.ru +figuringmoneyout.com +finansov.info +findpics.pw +findthe.pet +foodcrafts.website +fotoxxxru.com +franch.info +front.to +gallerylisting.com +gamesprite.me +gamewrath.com +gdzkurokam.ru +getoutofdebtfree.org +gheus.altervista.org +gigablast.com +globalscam.ga +gne8.com +go2album.com +google-liar.ru +googleseo.com.tr +graphics8.info +greatdealshop.com +greatzip.com +gsasearchengineranker.xyz +gsasearchenginerankerdiscount.com gta-top.ru -guiadeserraazul.com guidefs.ru -hdxnxxtube.mobi -help.tpu.ru -hitcpm.com -hledejvshopech.cz -hop.clickbank.net -hoztorg-opt.ru -hptwaakw.blog.fc2.com +hahashka.ru +hamptonoaks.ca +handicapvantoday.com +hard-porn.mobi +hawaiielectriclight.com +hazardky.net +hermesreplica.pw +herokuapp.com +hiwibyh.bugs3.com +hmywwogw.bloger.index.hr +hothor.se +howlongdoestizanidinestayinyoursystem.blogspot.com +howmuchdoestizanidinecost.blogspot.com hyipmanager.in -i-midias.net.br -igadgetsworld.com -igtools.club -ilo134uloh.com -imagerydatabase.com -img.wallpaperstock.net -indiakino.net -infokonkurs.ru -intelhdgraphicsgtdrive6w.metroblog.com -intellekt21.ru -interesnie-faktu.ru +idealtits.net +images-free.net +imgarit.pw +impisr.ru +innodgfdriverhm.aircus.com +instabid.tech +intermesh.net invest-pamm.ru -investsuccess.org -invivo.hu -iradiology.ru -iscblog.info -isotoner.com -itag.pw -itis4you.com -izismile.com -j33x.com -jerseyschinabizwholesale.us +iqoption.com +iqupdatetmz.win +istmira.ru +itservicesthatworkforyou.com +j-times.ru +jaxcube.info +jerseyssportsshop.com juliaworld.net -junketjuice.blogspot.com -kambasoft.com -kazan.xrus.org -keki.info -kenaba.su -kerei.ru -kinky-fetishes.com -kinoplen.ru -kit-opt.ru -kiwe-analytics.com -klitimg.pw +kandidos.com +karaganda.xkaz.org +kartiniresto.com +keenoutlet.online +keywesthideaways.co +keywordblocks.com +kidd.reunionwatch.com +kinoduh.ru +kirov.zrus.org +kiwi237au.tk +kletkimehan.ru klumba55.ru -kollekcioner.ru -koopilka.com -krasnodar.xrus.org -kvartira-sutochno.com -ladov.ru -languagecode.com -law-check-eight.xyz -law-check-two.xyz -lechenie-gemorroya.com -lexiangwan.com -littlesunraiser.com -llastbuy.ru -lomb.co -lookingglassemb.com +knowyournextmove.com +kostenlos-sexvideos.com +kovrenok.ru +kraljeva-sutjeska.com +krasnodar.zrus.org +kurbappeal.info +kursy-ege.ru +lacapilla.info +lapolis.it +lasvegaslockandsafe.com +lazy-z.com +lenpipet.ru +libertybilisim.com +lida-ru.com +linkredirect.biz +lion.conboy.us +livejasmin.com +liver-chintai.org +lodki-pvh.dp.ua lottospring.com -lowephotos.info -lumb.co -lyngdalhudterapi.no -mackeeper-center.club -manifestyourmillion.com -manualterap.roleforum.ru -marblestyle.ru -mbiologi.ru -mcar.in.ua -medkritika.ru -meetlocalchicks.com -megaindex.ru -mere.host.sk -mhi-systems.ru -michaelkorsoutlet.store -mini-modus.ru -missclub.info -misterjtbarbers.com +lovasszovetseg.hu +luckyshop.net.ua +m3gadownload.pl +macnewtech.com +macrotek.ru +mailemedicinals.com +maridan.com.ua +marmitaco.cat +mebel-vstroika.ru +medicovi.com +medmajor.ru +meds-online24.com +mensandals.xyz +mexicosleevegastrectomy.com +mini.7zap.com +mirzonru.net +mncrftpcs.com +mobicover.com.ua +mobile-appster.ru +mojaocena.com mompussy.net -monclerboots.xyz -moscow.online-podarki.com -moscow.xrus.org -mowser.com -moxo.com -mrsdalloways.com -mrvideospornogratis.xxx -my-big-family.com -mydirtyhobby.com -myfreemp3.eu -myfreetutorials.com +monsterdivx.com +monsterdivx.tv +mrbojikobi4.biz +mrwhite.biz +mulberryoutletonlineeu.com +mundoaberrante.com +muznachas-service.ru +mxgetcode.com +mypornfree.ru +narosty.com +netvouz.com +networkad.net newtechspb.ru -nfljerseyscheapchinabiz.com -nippon-bearings.ru -nisuturnetdgu.tk -nuup.info -o-o-6-o-o.com -oecnhs.info -okel.co -okroshki.ru -olgacvetmet.com -olympescort.com -onclkads.com -onetravelguides.com -onlinemeetingnow.com -openfrost.com -origin-my.ru -ostrovtaxi.ru -ourville.info -pacificair.com -pammik.ru -parfusale.se -partner-print.men -patterntrader-en.com -penisvergrotendepillennl.ovh -perfection-pleasure.ru -perso.wanadoo.es -pewit.pw +nhl17coins.exblog.jp +niki-mlt.ru +nikitsyringedrivelg.pen.io +novgorod.xrus.org +nzfilecloud.weebly.com +odywpjtw.bloger.index.hr +offer.camp +ok-ua.info +oksrv.com +onlinefilmz.net +onlineporno.site +ontargetseo.us +ooo-olni.ru +openfrost.net +outrageousdeal-a.akamaihd.net +oynat.info +panamaforbeginners.com +partner-cdn.men +partnerads.men +parvezmia.xyz +pb-dv.ru phelissota.xyz -photostudiolightings.com -picmoonco.pw -picurams.pw -pintattoos.com -pix-hd.com -pizza-tycoon.com -pladform.ru -poisk-zakona.ru +phormchina.com +photochki.com +pinsdaddy.com +pixgood.com +pkr1hand.com +plastgran.com +pontiacsolstice.info +popmarker.com +poppen-nw.net popugaychiki.com porn555.com -porngalleries.top pornhubforum.tk -porno-home365.com -pornohub.me -pornpost.in -predmety.in.ua +pornonik.com +pornoslive.net +pornotubs.com +pornozhara.com +portside.cc +pozdrawleniya.com +preconnubial.usuby.site +prefersurvey.net preventheadacheguide.info -producm.ru -profit-opportunity.com -pron.pro -prostitutki-rostova.org -psoriasis-file.trade -puserving.com +privatefx-in.ru +prize44.com +profolan.pl +prointer.net.ua +prom23.ru +prostitutki-belgoroda.org +prostitutki-novgoroda.org +purplestats.com +putevka24.ru putitin.me +py100.ru +pyramidlitho.webs.com quelle.ru -quickloanbank.com -radiodigital.co -ranking2017.ga -rapevideosmovies.com -rapidokbrain.com -rcb101.ru -recinziireale.com -redhotfreebies.co.uk -relayblog.com -remont-comp-pomosh.ru -remont-ustanovka-tehniki.ru -remote-dba.de -replicaclub.ru -reward-survey.net +radiogambling.com +randalljhoward.com +rangjued.com +rasteniya-vs-zombi.ru +redbottomheels.xyz +registry-cleaner.net +rentaremotecomputer.com rfserial.net -rockprogblog.com -running-line.ru -rus-teh.narod.ru -rusoft-zone.ru -russian-postindex.ru -rvtv.ru +romhacking.ru sale-japan.com -sarm.tk -sbtdesign.co.uk -scalerite.co.za -scanner-alexa.top -scanner-marwin.top -scat.porn -sdelatmebel.ru -searchengineranker.email -seeresultshub-a.akamaihd.net -seoexperimenty.ru -seorankinglinks.com -sethrollins.net -sex-foto.pw -sex-sex-sex5.com -sexstream.pl -sexvideo-sex.com -sexy-pings.com -sexyali.com +sampleletters.net +saugatuck.com +sax-sex.com +scanner-john.top +scanner-josh.top +scanner-olga.top +scanner-willy.top +scenarii-1-sentyabrya.uroki.org.ua +seansonline24.pl +search.1and1.com +searchtooknow-a.akamaihd.net +selectads.men +seo-prof1.xyz +seofied.com +seoprofiler.com +seorussian.ru +seotoolsagency.com +sergiorossistore.online +serving.adbetclickin.pink +servisural.ru +seryeznie-znakomstva.ru +sevendays.com.ua +sexsearch.com +sexyebonyteen.com sexytrend.ru -sherlock.se -shivafurnishings.com -shop-electron.ru -shopwme.ru -sideeffectsoftizanidine.blogspot.com -sientalyric.co -sim-service.net -simple-image.com.ua -sisiynas.ru -sitevalued.com -sklad-24.ru -small-game.com -smokewithrabbits.com -snowplanes.com -socialmediasuggest.com -socialseet.ru -softonicads.com -songoo.wz.cz -space-worry.ml +shakhtar-doneck.ru +shiksabd.com +siha.de +simply.net +sims-sims.ru +site-analyzer.com +site-auditor.online +sitebeam.net +sitopreferito.it +sivs.ru +skincrate.net +sky-mine.ru +sluganarodu.ru +smailik.org +small-games.biz +snip.tw +socialtrade.biz +sockshare.net +softomix.com +soviet-portal.do.am +speechfoodie.com sprttrack.com -srdrvp.com -srecorder.com -star61.de -start.myplaycity.com -startwp.org -sticken.co -stpolice.com -strangeduckfilms.com -stroyplus.ru -susanholtphotography.com +spywareit.com +srgwebmail.nl +stairliftstrue.com +steebook.com +sterva.cc +streetfooduncovered.com +stroimajor.ru +stroyhelp-dv.ru +sudexpert66.ru +sukarame.net +supers.com.ua +supervesti.ru +survival.betteroffers.review +suture.co svensk-poesi.com -swingerseiten.com -swiped.su -symphonyintegratedhealthcare.com -telefonsexsofort.tk -telegraf.by -template-kid.com -thecoral.com.br -theplacetoupdating.pw -thepornsex.org -timetorelax.biz -titangel-vietnam.com -titslove.yopoint.in -tnaionline.org -tomatis.gospartner.com +sygraem.com +tastyfoodideas.com +tech4master.com +teguh.info +tehngr.ru +thecounter.com +thedownloadfreeonlinegames.blogspot.com +themeforest.net +thin.me.pn +tivolibasket.it +tizanidinedrugtest.blogspot.com +tizanidinevscyclobenzaprine.blogspot.com +tn811.us +tnctrx.com +top-deal.com.pl +top-karkas.ru topads.men -topcasinoratings.ru -topshef.ru -tracksz.co -traderzplanet.in -trafficreceiver.club -trion.od.ua -trustl.life -tryrating.com -tubo360.com -tula.mdverey.ru -uchil.net -uggsale.online -um-razum.ru -unrealcommander.biz -usdx.us -ustion.ru -v24s.net -vashsvet.com -videojam.tv -vids18.site -vikistars.com -vip-dom.in -vip-parfumeria.ru -vitoriacabos.com -vkgaleria.com -vodaodessa.com -voprosotvet24.ru -vote-up.ru -vse-pesni.com -vykup-avto-krasnodar.ru +torture.ml +tourcroatia.co.uk +tpu.ru +tracfone.com +track2.shop +tradedeals.biz +traffpartners.com +ttsq.fr +tube8.com +tversvet.ru +typimga.pw +ucsol.ru +ukkelberg.no +ultimateclassicrock.com +url2image.com +urlopener.com +utiblog.fr +vadimkravtcov.ru +valmetrundan.se +vapomnoncri.tk +vbabule.net +vedomstvo.net +vektorpress.ru +videochat.cafe +videonsk.com +viteonlusarezzo.it +voloomoney.com +vostoktrade.info vykupavto-krasnodar.ru -wallpapers-best.com -watchinf.com -webnode.me -website-speed-check.site -wemedinc.com -westvilletowingservices.co.za -whereiskentoday.com -wineitudes.wordpress.com -wiosenny-bon-1500.pl -wladimirpayen.com -wma-x.com -workle.website -worldtraveler.world -wrona.it -wufak.com -www.souvenirua.com -www.timer4web.com -wygraj-teraz.com -xcombear.ru -xev.ru -xgames-04.com -xmnb.net -xn----8sbarihbihxpxqgaf0g1e.xn--80adxhks -xn----btbdvdh4aafrfciljm6k.xn--p1ai -xn--80aaafbn2bc2ahdfrfkln6l.xn--p1ai -xn--oogle-wmc.com -xtube.com -xxxasianporn.net -xxxguitars.com -y8games-free.com -yebocasino.com -youdesigner.kz -yousense.info -youtube-downloader.savetubevideo.com -zaloadi.ru -zapatosenventa.info -zebramart.ru -zero1.it -zx6.ru -zynax.ua +waterefficiency.co +waycash.net +web-betting.ru +webenlace.com.ar +weblo.com +website-stealer.nufaq.com +whatistizanidine2mg.blogspot.com +white-truck.life +whiteelephantwellington.com +wikes.20fr.com +winner7777.net +wordkeyhelper.com +writersgroup580.web.fc2.com +www.888.com +x5market.ru +xchangetrak.com +xn----7sbaaabaei0cc8aj5bj0bncejx.xn--p1ai +xn----itbeirbjbi7bc6bh2d.xn--p1ai +xn--80aanaardaperhcem4a6i.com +yaderenergy.ru +youghbould.wordpress.com +yoursite.com +youtoner.it +youtubedownload.org +yurgorod.ru +zahvat.ru +zaidia.xhost.ro +zakazfutbolki.com +zakon-ob-obrazovanii.ru +zakonobosago.ru +zeikopay.com +zerocash.msk.ru +zeroredirect.com +zeroredirect5.com +zigarettenonl.canalblog.com +znakom.sibtest.ru diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index a2ca9d4e2..cf031575d 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -01apple.com -123movies.love -23kw.ru -24videos.tv -3rbseyes.com -5forex.ru -98oi.ru -abwa.tk -adminshop.com -adservme.com -adultnet.in -agecheckadult.com -alfa-medosmotr.ru -alfabot.xyz -allwomen.info -amung.us -anatomiy.com -angkortours.vn -apifasterlightin-a.akamaihd.net -arcadeplayhouse.com +00webcams.com +1008.su +123locker.com +12u.info +3-letter-domains.net +51unlim.ru +66cpwgln.space +adinterax.com +adultwebhosting.info +advertisingtag.net +agahinameh.com +agreda.pluto.ro +allboard.xobor.de +alta-realestate.com +altamayoztourism.com +an-donut.com +apartment.ru +apelsinnik.website +atdedinotuho.tk +atofilms.com audiobangout.com -auto-moto-elektronika.cz -barnfurnituremart.com -beclean-nn.ru -best-games.today -binomo.kz -bmw-ark.ru -bodybuilding-shop.biz -bukleteg.xyz -bumble.cheapwebsitehoster.com -buy-meds24.com -buyantiviralwp.com -carabela.com.do -cartierlove.xyz -cartierlovestore.com -cdnanalytics.xyz -checkhit.com -chinajerseyswholesalecoupons.com -chinese-amezon.com -cibpenonptib.flu.cc -clean-start.net -clicksor.net -clubfashionista.com -coderstate.com -coin-hive.com -copblock.org -copyrightclaims.org -counterbot.com -courtshipgift.com -customboxes4less.com -customcollegeessays.net -cutalltheshit.com -darkbooks.org -dignitasdata.se -directrev.com -diy-handmade-ideas.com -djihispano.com -dneprsvet.com.ua -domain2008.com -domaincheck.io -drillsaw.ru -dscaas.website -dumpsmania24.com -dwomlink.info -egovaleo.it -enternet.ee -erotik0049.com +autoblogger24.info +ayanaspa.com +azartniy-bonus.com +bankmib.ru +beauty-bracelet.com +beetpics.pw +bestcoin.cc +besthoro.ru +bestmouthwash.club +bestofpicture.com +betterhdporn.com +beyan.host.sk +bezcmexa.ru +bikini-image.com +bimatoprost-careprost.com +boanonihaca.tk +bookmark4you.com.biz +bpro1.top +break-the-chains.com +buzzzg.men +cakesplus.com.au +cartechnic.ru +cementaresearch.se +certifywebsite.win +cfacarrosserie74.com +cheapcigarettesc.info +chuckguilford.com +cialisovercounteratwalmartusa.com +cigarpass.com +clicksor.com +cntravelre.com +cometorussia.net +compliance-andrew.top +compliance-jack.top +consorzioilmosaico.org +contentlook.co +cosmeticswomens-womensports.rhcloud.com +davebestdeals.com +dbmkfhqk.bloger.index.hr +diminishedvalueoforegon.com +dirtpics.pw +documentsite.net +dostavimvdom.ru +drugstoreforyou.com +ebonyporn.site +ebooknovel.club +elektrozigarette2.webs.com +elektrozigaretten2.yn.lt essayassist.com -everypony.ru -eyemagination.com -ezigarettenshop2.postbit.com -f-loaded.de -f00kclan.de -fashion-stickers.ru -fickenbumsen.net +europages.com.ru +ezigarettekaufen2.blox.pl +familyholiday.ml +faptitans.com +findthe.pet +flytourisme.org +forum20.smailik.org foxweber.com -free-social-buttons.com -freenom.link -frivgame250.com -front.to -fungirlsgames.net -fusoradio.info -galleryawesome.com -gate5.co.za -gaypornmovie.net -genericlowlatencyasiodriverhq.aircus.com -get-seo-domain.com -gifspics.com -gigablast.com -girlsfuckdick.com -golden-praga.ru -greatdealshop.com -gsasearchengineranker.space -gta-club.ru +free-fb-traffic.com +free-gluten.ru +freelifetimefuckbook.com +frequiry.com +freshdz.com +froggytube.com +gamedayassist.com +ge0ip.com +goodwinmetals.co +gorodservis.ru +gpms.org.my +greenzaim.ru gta-top.ru -hd720kino.ru -hobbyhuren-datenbank.com -hoholikik.club -hostnow.men -hothot.ru -house.sieraddns.com -hully.altervista.org -hunthillfarmtrust.org -hut1.ru -i-midias.net.br -imagefinder.site -imicrovision.com -inbabes.sexushost.com -incep.imagine-ex.co +hdpixion.com +help.tpu.ru +heroz.fr +hmmm.cz +ie.57883.net +igrovyeavtomaty777.ru +ilo134uloh.com +imfamous.info infogame.name -insta-add.pro -ivanovo.zrus.org -iweblist.info -keywordhut.com -kinogonew.ru -kiwi237au.tk -kotaku.com -koversite.info -kovesszucs.atw.hu -kovrenok.ru -ks1234.com -ktotut.net -kumuk.info -lancheck.net -law-check-seven.xyz -liky.co.ua -linkbux.ru -linkdebrideur.xyz -linkredirect.biz -littlesunraiser.com -liveinternet.ru -lizace.com -lodki-pvh.dp.ua -magnetic-bracelets.ru -magnetpress.sk -marketingvici.com -meddesk.ru -menstennisforums.com -mnogabukaff.net -moesonce.com -mouselink.co -msfsaar.de -mttwtrack.com -muschisexbilder.com -mustwineblog.com -mydeathspace.com -naval.jislaaik.com -neobux-bg.info -nero-us.com -news-speaker.com -nomuos.it -ofanda.com -onemactrckr.com -onlyporno.ru -ontargetseo.us -paparazzistudios.com.au -parajumpersjakkesalgnorge.info -partnerads.men -pavlodar.xkaz.org -pb-dv.ru -peekyou.com -pestomou.info -photokitchendesign.com -picturesfrom.com -pinup-casino1.ru -pixgood.com -pixshark.com -pizdopletka.club -pobeiranie.pl -pornosok.ru -powc.r.ca.d.sendibm2.com -pozdrawleniya.ru -privacyassistant.net -privetsochi.ru -prizefestival.mobi -promalp-universal.ru -prostitutki-rostova.org -quizzitch.net -qwarckoine.com -ranksonic.info -rebrand.ly -registrysweeper.com -responsive-test.net -rock-cafe.info -rockprogblog.com -scanspyware.net -scoopquest.com -sdrescher.net -se.bnt-team.com -seemoreresultshu-a.akamaihd.net -seksotur.ru -seo-tools-optimizing.com -seokicks.de -serptehnika.ru -servisural.ru -sharebutton.net -shemalegalls.blogporn.in -shitmovs.com -shoesonlinebuy.cn -sitiz.club -smartshoppymac.com -spammen.de -squidoo.com -srgwebmail.nl -stardevine.com -strfls.com -studiofmp.com -surcentro.com -suture.co -svarbit.com -textads.men -tizanidinenarcotic.blogspot.com -tnaionline.org -toy-shop.top -trafficfactory.biz -trafficstars.com -trichizobswiv.agddns.net -trucri.me -tvnewsclips.info -twitlinks.com -twu.com.ua -uginekologa.com -ugogo.info -uptimebot.net -url-extractor.xyz -usacasino.com -valueclick.com -vapomnoncri.tk -vbtracker.net -vekzdorov.ru -vepad.com -via-energy-order.com -viagra.pp.ua -vinylvault.co.uk -voloo.ru -voltrknc1.com -vseigry.fun -wahicbefa31.soup.io -website-stealer.nufaq.com -whatzmyip.net -wma-x.com -wowas31.ucoz.ru -www.event-tracking.com -xaijo.com -yaderenergy.ru -yeartwit.com -yellocloud.be -yoopsie.com -yourothersite.com -zrizvtrnpale.tk +integritylandscapeservices.com +internet-apteka.ru +intervsem.ru +iqupdatetmz.win +itis4you.com +jackwolfskinoutlet.online +janerikholst.se +janettabridal.com +jetsli.de +jyrxd.com +karadene.com +kartiniresto.com +keywordglobal.co.za +keywordsking.com +khadastoafarde.tk +lafourmiliaire.com +ldrtrack.com +leadwayau.com +linerdrilling.com +lookingglassemb.com +louboutinreplica.xyz +luchshie-topcasino.ru +macrotek.ru +mainlinehobby.net +manimpotence.com +mantramusic.ru +max-p.men +med-zdorovie.com.ua +mediaoffers.click +medictube.ru +medmajor.ru +metascephe.com +migente.com +mmgq.ru +mobicover.com.ua +mobile-appster.ru +moi-glazki.ru +monsterdivx.com +montazhnic.ru +moonci.ru +mug-na-chas-moscow.ru +myxdate.info +na-telefon.biz +napalm51.nut.cc +nicovideo.jp +norththeface.store +notasprensa.info +nudejapan.net +nudo.ca +office2web.com +offtopic.biz +oivcvx.website +omoikiri-japan.ru +onstrapon.purplesphere.in +oops-cinema.ru +openfrost.net +orion-code-access.net +owathemes.com +p-business.ru +panouri-solare-acoperis.com +patterntrader-en.com +photochki.com +pk-services.ru +plastjulgranar.se +poem-paying.gq +polytopesexempt.com +porno-transsexuals.ru +pornokorol.com +pornozhara.com +potoideas.us +princevc.com +prointer.net.ua +qpypcx.com +quebec-bin.com +randki-sex.com +recipedays.ru +respublica-otel.ru +rfserial.net +riders.ro +roleforum.ru +royal-betting.net +rucrypt.com +rumamba.com +rvtv.ru +saddiechoua.com +saitevpatorii.com +savetubevideo.com +scanner-nelson.top +seomarketings.online +serving.adbetclickin.pink +sethrollins.net +sexkrasivo.net +shakhtar-doneck.ru +shariki-zuma-lines.ru +sheerseo.com +ship-marvel.co.ua +silvermature.net +simple-image.com.ua +slavic-magic.ru +social-s-iii.xyz +soft1.ru +startwp.org +stmassage.ru +supermodni.com.ua +swagbucks.com +telegraf.by +telsis.com +todohr.com +tomck.com +top10registrycleaners.com +track-rankings.online +trubywriting.com +tube8.com +tubeline.biz +tubo360.com +tx41tclega.ru +udayavani.com +urldelivery.com +velen.io +vgoloveboli.net +vielporno.net +vietimgy.pw +vilingstore.net +vkontaktemusic.ru +vsesubwaysurfers.com +vykup-avto-krasnodar.ru +wallpapers-best.com +wapsite.me +watchinf.com +wdrake.com +wesharepics.com +wildcattube.com +wsgames.ru +x-porno.video +xgames-04.com +xn-----6kcaabbafhu7cskl7akvongwpo7hvjj.xn--p1ai +xn----8sbhefaln6acifdaon5c6f4axh.xn--p1ai +xn--1-8sbcpb0bdm8k6a.xn--p1ai +xn--90acenikpebbdd4f6d.xn--p1ai +xn--e1afanlbnfckd7c3d.xn--p1ai +xn--l1aengat.xn--p1ai +xnxx-n.com +xtube.com +xwatt.ru +yoshkarola.zrus.org +yougame.biz +youradulthosting.com +yun56.co +zeroredirect12.com +zeroredirect9.com diff --git a/dev-tools/test_units/ratelimittest-beta.txt b/dev-tools/test_units/ratelimittest-beta.txt index 3656720f2..f0474a154 100644 --- a/dev-tools/test_units/ratelimittest-beta.txt +++ b/dev-tools/test_units/ratelimittest-beta.txt @@ -581,14 +581,4 @@

Welcome to the NginX Ultimate Bad Bot Blocker

This is merely used for testing !!!

- - - -Welcome to NginX - - -
-

Welcome to the NginX Ultimate Bad Bot Blocker

-

This is merely used for testing !!!

- \ No newline at end of file diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index e0a7c466d..92ab0df4c 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.217 0; +10.1.1.31 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index 5997d586e..4567ab16e 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -47400,3 +47400,1253 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2489 +### Updated: Thu Jul 29 11:09:37 UTC 2021 +### Bad Bot Count: 616 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ALittle Client +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: ubermetrics-technologies.com +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ + diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 5997d586e..4567ab16e 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -47400,3 +47400,1253 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2489 +### Updated: Thu Jul 29 11:09:37 UTC 2021 +### Bad Bot Count: 616 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ALittle Client +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: ubermetrics-technologies.com +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ +