From 1db74d9256e3b5844c2a28853ab6e0efe430baf0 Mon Sep 17 00:00:00 2001 From: mitchellkrogza Date: Thu, 29 Jul 2021 11:37:16 +0000 Subject: [PATCH] V4.2021.07.2490 [ci skip] --- AUTO-CONFIGURATION.md | 4 +- MANUAL-CONFIGURATION.md | 4 +- README.md | 4 +- _generator_lists/bad-user-agents.list | 2 +- _google_analytics_ghost_spam/README.md | 4 +- .../etc/nginx/conf.d/globalblacklist.conf | 7 +- conf.d/globalblacklist-testing-version.conf | 7 +- conf.d/globalblacklist.conf | 7 +- dev-tools/_robots_input/robots-input.txt | 1 + .../conf.d/globalblacklist.conf | 7 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 7 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 7 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/blacklist-user-agents.conf | 1 + .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 7 +- dev-tools/buildnumber | 2 +- dev-tools/globalblacklist-testing.template | 7 +- dev-tools/globalblacklist.template | 7 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 7 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 7 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 7 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/blacklist-user-agents.conf | 1 + .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 7 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 7 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 7 +- dev-tools/test_units/blacklist-ips.conf | 2 +- .../test_units/blacklist-user-agents.conf | 1 + .../random-bots-for-test-quick.list | 20 +- .../test_units/random-bots-for-test.list | 314 ++--- .../random-bots-for-whitelist-test.list | 284 ++-- .../random-referrers-for-test-quick.list | 20 +- .../test_units/random-referrers-for-test.list | 924 ++++++------ .../random-referrers-for-whitelist-test.list | 480 +++---- dev-tools/test_units/ratelimittest-beta.txt | 10 + dev-tools/test_units/whitelist-ips.conf | 2 +- dev-tools/tmp/robots.txt | 1252 +++++++++++++++++ robots.txt/robots.txt | 1252 +++++++++++++++++ 51 files changed, 3626 insertions(+), 1093 deletions(-) diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 8085c0a67..b232e5d4e 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2489 +#### Version: V4.2021.07.2490 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [616](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [617](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index abeacd106..9e01421a6 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,9 +4,9 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2489 +#### Version: V4.2021.07.2490 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [616](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [617](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/README.md b/README.md index e2ea36367..cb1e38750 100755 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2021.07.2489 +#### Version: V4.2021.07.2490 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [616](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [617](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_generator_lists/bad-user-agents.list b/_generator_lists/bad-user-agents.list index 35fffebab..9354cad9a 100755 --- a/_generator_lists/bad-user-agents.list +++ b/_generator_lists/bad-user-agents.list @@ -1,4 +1,3 @@ -Leap 01h4x.com 360Spider 404checker @@ -224,6 +223,7 @@ LNSpiderguy LWP::Simple Lanshanbot Larbin +Leap LeechFTP LeechGet LexiBot diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 6b329923b..677e5bf03 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,9 +3,9 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2021.07.2489 +#### Version: V4.2021.07.2490 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) -#### Bad User-Agents (Bots) Blocked: [616](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) +#### Bad User-Agents (Bots) Blocked: [617](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) ____________________ diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index f97b2e17d..4bf7536e4 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2489 -### Updated: Thu Jul 29 11:09:36 UTC 2021 +### Version: V4.2021.07.2490 +### Updated: Thu Jul 29 11:31:24 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 616 +### Bad Bot Count: 617 ################################################### ### VERSION INFORMATION ## @@ -372,6 +372,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LWP::Simple(?:\b)" 3; "~*(?:\b)Lanshanbot(?:\b)" 3; "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)Leap(?:\b)" 3; "~*(?:\b)LeechFTP(?:\b)" 3; "~*(?:\b)LeechGet(?:\b)" 3; "~*(?:\b)LexiBot(?:\b)" 3; diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 888ce68f2..af249ef8b 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2489 -### Updated: Thu Jul 29 11:09:37 UTC 2021 +### Version: V3.2021.07.2490 +### Updated: Thu Jul 29 11:31:25 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 616 +### Bad Bot Count: 617 ################################################### ### VERSION INFORMATION ## @@ -372,6 +372,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LWP::Simple(?:\b)" 3; "~*(?:\b)Lanshanbot(?:\b)" 3; "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)Leap(?:\b)" 3; "~*(?:\b)LeechFTP(?:\b)" 3; "~*(?:\b)LeechGet(?:\b)" 3; "~*(?:\b)LexiBot(?:\b)" 3; diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index f97b2e17d..4bf7536e4 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2489 -### Updated: Thu Jul 29 11:09:36 UTC 2021 +### Version: V4.2021.07.2490 +### Updated: Thu Jul 29 11:31:24 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 616 +### Bad Bot Count: 617 ################################################### ### VERSION INFORMATION ## @@ -372,6 +372,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LWP::Simple(?:\b)" 3; "~*(?:\b)Lanshanbot(?:\b)" 3; "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)Leap(?:\b)" 3; "~*(?:\b)LeechFTP(?:\b)" 3; "~*(?:\b)LeechGet(?:\b)" 3; "~*(?:\b)LexiBot(?:\b)" 3; diff --git a/dev-tools/_robots_input/robots-input.txt b/dev-tools/_robots_input/robots-input.txt index 6adaaaad2..4069cf282 100644 --- a/dev-tools/_robots_input/robots-input.txt +++ b/dev-tools/_robots_input/robots-input.txt @@ -223,6 +223,7 @@ LNSpiderguy LWP::Simple Lanshanbot Larbin +Leap LeechFTP LeechGet LexiBot diff --git a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 888ce68f2..af249ef8b 100644 --- a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2489 -### Updated: Thu Jul 29 11:09:37 UTC 2021 +### Version: V3.2021.07.2490 +### Updated: Thu Jul 29 11:31:25 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 616 +### Bad Bot Count: 617 ################################################### ### VERSION INFORMATION ## @@ -372,6 +372,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LWP::Simple(?:\b)" 3; "~*(?:\b)Lanshanbot(?:\b)" 3; "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)Leap(?:\b)" 3; "~*(?:\b)LeechFTP(?:\b)" 3; "~*(?:\b)LeechGet(?:\b)" 3; "~*(?:\b)LexiBot(?:\b)" 3; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index de65f07dd..5eea2512c 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.31 1; +10.1.0.149 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 92ab0df4c..338f1d6f7 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.31 0; +10.1.0.149 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 888ce68f2..af249ef8b 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2489 -### Updated: Thu Jul 29 11:09:37 UTC 2021 +### Version: V3.2021.07.2490 +### Updated: Thu Jul 29 11:31:25 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 616 +### Bad Bot Count: 617 ################################################### ### VERSION INFORMATION ## @@ -372,6 +372,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LWP::Simple(?:\b)" 3; "~*(?:\b)Lanshanbot(?:\b)" 3; "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)Leap(?:\b)" 3; "~*(?:\b)LeechFTP(?:\b)" 3; "~*(?:\b)LeechGet(?:\b)" 3; "~*(?:\b)LexiBot(?:\b)" 3; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index de65f07dd..5eea2512c 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.31 1; +10.1.0.149 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 92ab0df4c..338f1d6f7 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.31 0; +10.1.0.149 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 888ce68f2..af249ef8b 100644 --- a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2489 -### Updated: Thu Jul 29 11:09:37 UTC 2021 +### Version: V3.2021.07.2490 +### Updated: Thu Jul 29 11:31:25 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 616 +### Bad Bot Count: 617 ################################################### ### VERSION INFORMATION ## @@ -372,6 +372,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LWP::Simple(?:\b)" 3; "~*(?:\b)Lanshanbot(?:\b)" 3; "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)Leap(?:\b)" 3; "~*(?:\b)LeechFTP(?:\b)" 3; "~*(?:\b)LeechGet(?:\b)" 3; "~*(?:\b)LexiBot(?:\b)" 3; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index de65f07dd..5eea2512c 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.31 1; +10.1.0.149 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 1cee28ac6..4b14dbd12 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -284,6 +284,7 @@ "~*(?:\b)LWP::Simple(?:\b|)" 0; "~*(?:\b)Lanshanbot(?:\b|)" 0; "~*(?:\b)Larbin(?:\b|)" 0; + "~*(?:\b)Leap(?:\b|)" 0; "~*(?:\b)LeechFTP(?:\b|)" 0; "~*(?:\b)LeechGet(?:\b|)" 0; "~*(?:\b)LexiBot(?:\b|)" 0; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 92ab0df4c..338f1d6f7 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.31 0; +10.1.0.149 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 888ce68f2..af249ef8b 100644 --- a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2489 -### Updated: Thu Jul 29 11:09:37 UTC 2021 +### Version: V3.2021.07.2490 +### Updated: Thu Jul 29 11:31:25 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 616 +### Bad Bot Count: 617 ################################################### ### VERSION INFORMATION ## @@ -372,6 +372,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LWP::Simple(?:\b)" 3; "~*(?:\b)Lanshanbot(?:\b)" 3; "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)Leap(?:\b)" 3; "~*(?:\b)LeechFTP(?:\b)" 3; "~*(?:\b)LeechGet(?:\b)" 3; "~*(?:\b)LexiBot(?:\b)" 3; diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index 697ade4bd..bf6937821 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -2489 +2490 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index 888ce68f2..af249ef8b 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2489 -### Updated: Thu Jul 29 11:09:37 UTC 2021 +### Version: V3.2021.07.2490 +### Updated: Thu Jul 29 11:31:25 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 616 +### Bad Bot Count: 617 ################################################### ### VERSION INFORMATION ## @@ -372,6 +372,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LWP::Simple(?:\b)" 3; "~*(?:\b)Lanshanbot(?:\b)" 3; "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)Leap(?:\b)" 3; "~*(?:\b)LeechFTP(?:\b)" 3; "~*(?:\b)LeechGet(?:\b)" 3; "~*(?:\b)LexiBot(?:\b)" 3; diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index f97b2e17d..4bf7536e4 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2489 -### Updated: Thu Jul 29 11:09:36 UTC 2021 +### Version: V4.2021.07.2490 +### Updated: Thu Jul 29 11:31:24 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 616 +### Bad Bot Count: 617 ################################################### ### VERSION INFORMATION ## @@ -372,6 +372,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LWP::Simple(?:\b)" 3; "~*(?:\b)Lanshanbot(?:\b)" 3; "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)Leap(?:\b)" 3; "~*(?:\b)LeechFTP(?:\b)" 3; "~*(?:\b)LeechGet(?:\b)" 3; "~*(?:\b)LexiBot(?:\b)" 3; diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 6a2d88156..a02ff7600 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -52.170.156.225 0; +13.66.166.149 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index f97b2e17d..4bf7536e4 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2489 -### Updated: Thu Jul 29 11:09:36 UTC 2021 +### Version: V4.2021.07.2490 +### Updated: Thu Jul 29 11:31:24 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 616 +### Bad Bot Count: 617 ################################################### ### VERSION INFORMATION ## @@ -372,6 +372,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LWP::Simple(?:\b)" 3; "~*(?:\b)Lanshanbot(?:\b)" 3; "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)Leap(?:\b)" 3; "~*(?:\b)LeechFTP(?:\b)" 3; "~*(?:\b)LeechGet(?:\b)" 3; "~*(?:\b)LexiBot(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index de65f07dd..5eea2512c 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.31 1; +10.1.0.149 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 92ab0df4c..338f1d6f7 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.31 0; +10.1.0.149 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index f97b2e17d..4bf7536e4 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2489 -### Updated: Thu Jul 29 11:09:36 UTC 2021 +### Version: V4.2021.07.2490 +### Updated: Thu Jul 29 11:31:24 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 616 +### Bad Bot Count: 617 ################################################### ### VERSION INFORMATION ## @@ -372,6 +372,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LWP::Simple(?:\b)" 3; "~*(?:\b)Lanshanbot(?:\b)" 3; "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)Leap(?:\b)" 3; "~*(?:\b)LeechFTP(?:\b)" 3; "~*(?:\b)LeechGet(?:\b)" 3; "~*(?:\b)LexiBot(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index de65f07dd..5eea2512c 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.31 1; +10.1.0.149 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 92ab0df4c..338f1d6f7 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.31 0; +10.1.0.149 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index f97b2e17d..4bf7536e4 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2489 -### Updated: Thu Jul 29 11:09:36 UTC 2021 +### Version: V4.2021.07.2490 +### Updated: Thu Jul 29 11:31:24 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 616 +### Bad Bot Count: 617 ################################################### ### VERSION INFORMATION ## @@ -372,6 +372,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LWP::Simple(?:\b)" 3; "~*(?:\b)Lanshanbot(?:\b)" 3; "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)Leap(?:\b)" 3; "~*(?:\b)LeechFTP(?:\b)" 3; "~*(?:\b)LeechGet(?:\b)" 3; "~*(?:\b)LexiBot(?:\b)" 3; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index de65f07dd..5eea2512c 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.31 1; +10.1.0.149 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf index 1cee28ac6..4b14dbd12 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-user-agents.conf @@ -284,6 +284,7 @@ "~*(?:\b)LWP::Simple(?:\b|)" 0; "~*(?:\b)Lanshanbot(?:\b|)" 0; "~*(?:\b)Larbin(?:\b|)" 0; + "~*(?:\b)Leap(?:\b|)" 0; "~*(?:\b)LeechFTP(?:\b|)" 0; "~*(?:\b)LeechGet(?:\b|)" 0; "~*(?:\b)LexiBot(?:\b|)" 0; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 92ab0df4c..338f1d6f7 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.31 0; +10.1.0.149 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index f97b2e17d..4bf7536e4 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2489 -### Updated: Thu Jul 29 11:09:36 UTC 2021 +### Version: V4.2021.07.2490 +### Updated: Thu Jul 29 11:31:24 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 616 +### Bad Bot Count: 617 ################################################### ### VERSION INFORMATION ## @@ -372,6 +372,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LWP::Simple(?:\b)" 3; "~*(?:\b)Lanshanbot(?:\b)" 3; "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)Leap(?:\b)" 3; "~*(?:\b)LeechFTP(?:\b)" 3; "~*(?:\b)LeechGet(?:\b)" 3; "~*(?:\b)LexiBot(?:\b)" 3; diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 6a2d88156..a02ff7600 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -52.170.156.225 0; +13.66.166.149 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 9ee9d9ecb..e2de7d9c5 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2489 -### Updated: Thu Jul 29 11:09:36 UTC 2021 +### Version: V4.2021.07.2490 +### Updated: Thu Jul 29 11:31:24 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 616 +### Bad Bot Count: 617 ################################################### ### VERSION INFORMATION ## @@ -372,6 +372,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LWP::Simple(?:\b)" 3; "~*(?:\b)Lanshanbot(?:\b)" 3; "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)Leap(?:\b)" 3; "~*(?:\b)LeechFTP(?:\b)" 3; "~*(?:\b)LeechGet(?:\b)" 3; "~*(?:\b)LexiBot(?:\b)" 3; diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 6a2d88156..a02ff7600 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -52.170.156.225 0; +13.66.166.149 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 5dd0ca518..12580c5d1 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,10 +4,10 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2489 -### Updated: Thu Jul 29 11:09:36 UTC 2021 +### Version: V4.2021.07.2490 +### Updated: Thu Jul 29 11:31:24 UTC 2021 ### Bad Referrer Count: 7068 -### Bad Bot Count: 616 +### Bad Bot Count: 617 ################################################### ### VERSION INFORMATION ## @@ -372,6 +372,7 @@ map $http_user_agent $bad_bot { "~*(?:\b)LWP::Simple(?:\b)" 3; "~*(?:\b)Lanshanbot(?:\b)" 3; "~*(?:\b)Larbin(?:\b)" 3; + "~*(?:\b)Leap(?:\b)" 3; "~*(?:\b)LeechFTP(?:\b)" 3; "~*(?:\b)LeechGet(?:\b)" 3; "~*(?:\b)LexiBot(?:\b)" 3; diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index de65f07dd..5eea2512c 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.31 1; +10.1.0.149 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/blacklist-user-agents.conf b/dev-tools/test_units/blacklist-user-agents.conf index 1cee28ac6..4b14dbd12 100644 --- a/dev-tools/test_units/blacklist-user-agents.conf +++ b/dev-tools/test_units/blacklist-user-agents.conf @@ -284,6 +284,7 @@ "~*(?:\b)LWP::Simple(?:\b|)" 0; "~*(?:\b)Lanshanbot(?:\b|)" 0; "~*(?:\b)Larbin(?:\b|)" 0; + "~*(?:\b)Leap(?:\b|)" 0; "~*(?:\b)LeechFTP(?:\b|)" 0; "~*(?:\b)LeechGet(?:\b|)" 0; "~*(?:\b)LexiBot(?:\b|)" 0; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 752a4dbbf..5f52398a3 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -CensysInspect -Demon -Id-search -IndeedBot -NetMechanic -SiteCheckerBotCrawler -Web Collage -muhstik-scan -sexsearcher -x22Mozilla +BBBike +Cliqzbot +Cloud mapping +Curious +Custo +Fyrebot +Hloader +Mozlila +Offline Navigator +SEOprofiler diff --git a/dev-tools/test_units/random-bots-for-test.list b/dev-tools/test_units/random-bots-for-test.list index c7b0e1c34..03b100416 100644 --- a/dev-tools/test_units/random-bots-for-test.list +++ b/dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -01h4x.com -AiHitBot +360Spider +80legs +ADmantX +AIBOT +ALittle Client +Aboundex +Acunetix +AfD-Verbotsverfahren Aipbot -Alexibot -AllSubmitter -Anarchy -Anarchy99 -Ankit -Apexoo -Aspiegel +Anarchie +Anthill Attach -AwarioSmartBot -BDFetch -BLEXBot -Barkrowler -BatchFTP -Battleztar Bazinga -Bigfoot +Backlink-Ceck +BacklinkCrawler +Badass Bitacle -Black Hole -Blow +Blackboard +Boardreader +Brandprotect +Buck +Buddy BuiltBotTough BuiltWith -Bullseye BuzzSumo -CODE87 -CSHttp -CheTeam -CheeseBot -Chlooe +CCBot +Calculon +Cegbfeieh +CherryPicker +ChinaClaw Claritybot -Cloud mapping +Cocolyzebot Copier +CopyRightCheck Cosmos -CrazyWebCrawler +Crescent CrunchBot Curious -Custo -CyotekWebCopy -DIIbot +DBLBot +DSearch DTS Agent -DatabaseDriverMysqli -Demon -Digincore -Dispatch +Devil +Dirbuster +Discoverybot +DittoSpyder DnyzBot -DomCopBot -DomainAppender -DomainSigmaCrawler -DomainStatsBot -Domains Project -Dotbot -Drip -ECCP/1.0 -EMail Wolf +DomainCrawler +Dragonfly +EMail Siphon EasyDL -Ecxi EirGrabber -EroCrawler Evil -Exabot -Express WebPictures +ExtLinksBot Extractor -Extreme Picture Finder -EyeNetIE -Ezooms +ExtractorPro +FDM +FemtosearchBot +Fimap Freeuploader -FyberSpider -Fyrebot +G-i-g-a-b-o-t GT::WWW -GalaxyBot -GermCrawler GetRight -GetWeb -Getintent -Gotit -GrabNet +Go-Ahead-Got-It +Grabber Grafula GrapeFX -HMView -HTMLparser -HTTrack +GridBot +Haansoft HaosouSpider -Heritrix -Hloader +Harvest Humanlinks -HybridBot -IDBTE4M -Id-search +IRLbot +Iblog +IlseBot Image Fetch -Image Sucker -InternetSeer +Indy Library +InfoTekies +Internet Ninja Iskanie -JOC Web Spider +IstellaBot +Jbrofuzz JennyBot +JetCar +Joomla JustView -Jyxobot -Kenjin Spider -Kinza +Keyword Density Kozmosbot Lanshanbot -LeechFTP +Larbin LeechGet -Lftp -LibWeb +Libwhisker +Likse +LinkScan +Linkdexbot LinkextractorPro -LinkpadBot -LinksManager -LinqiaMetadataDownloaderBot -LinqiaScrapeBot -Lipperhey -Lipperhey Spider -Litemage_walker -MIDown tool -MIIxpc -MJ12bot -MQQBrowser -MTRobot -Majestic SEO +LinqiaRSSBot +Lmspider +MFC_Tear_Sample +Magnet +Mail.RU_Bot Majestic-SEO -MauiBot -Mb2345Browser -Mediatoolkitbot +MarkWatch +Mata Hari +MegaIndex.ru +Metauri MicroMessenger +Mister PiX Moblie Safari Mojeek Mojolicious -NICErsPRO +Morfeus Fucking Scanner +Mozlila +Mr.4x3 +Msrabot Navroad -Nessus Net Vampire NetAnts -NetLyzer NetMechanic -NextGenSearchBot +NetSpider +NetZIP +Netvibes Nibbler Niki-bot -Nuclei -Nutch +Nikto +Octopus +OnCrawl OpenLinkProfiler -Openfind Openvas -OrangeBot -PECL::HTTP +OrangeSpider +OutclicksBot +OutfoxBot +POE-Component-Client-HTTP +Page Analyzer PageAnalyzer -PageThing.com -Pandalytics -PeoplePal -Petalbot -Picscout +PageScorer +Papa Foto +Pavuk +Pi-Monster Picsearch -PictureFinder +Pimonster Pockey +ProPowerBot ProWebWalker Probethenet -Psbot Pu_iN -PxBroker -Quick-Crawler -RSSingBot +PyCurl +QueryN Metasearch +RankActive +RankActiveLinkBot RankFlex -RankingBot2 -Re-re -ReGet -RealDownload -RebelMouse +RankingBot +Rankivabot +RankurBot Recorder +Rogerbot SBIder +SISTRIX SMTBot SalesIntelligent +ScanAlert Scrapy -Screaming -ScreenerBot -SearchmetricsBot -Seomoz -Shodan -SiteExplorer -SiteLockSpider -Sitebeam -SlySearch +ScrepyBot +Searchestate +SentiBot +Siphon +SiteCheckerBotCrawler +Sitevigil +SmartDownload Snapbot -Sociscraper -SpaceBison +SocialRankIOBot +Sosospider Spanner -SputnikBot -Sqworm -Steeler +SuperHTTP +Surfbot Suzuran Swiftbot Szukacz -TeleportPro +Teleport +Telesoft The Intraformant -TightTwatBot -Toweyabot -Tracemyfile +Thumbor +Titan Trendictionbot -True_Robot +Turnitin +TurnitinBot TwengaBot +URLy.Warning Upflow -Vacuum -VeriCiteCrawler -WBSearchBot -WPScan +VCI +Vagabondo +Virusdie +WASALive-Bot +WISENutbot +WWW::Mechanize +Wallpapers/3.0 Web Auto Web Collage +Web Enhancer +Web Fetch Web Pix -Web Sauger Web Sucker +WebAuto WebCollage -WebFuck +WebFetch +WebImageCollector WebPix +WebReaper +WebSauger +WebStripper +WebWhacker +WebZIP Webalta Webshag Website Quester WebsiteExtractor -Whacker -Who.is Bot +Whatweb WinHTTrack WiseGuys Robot -Wonderbot Woobot +Wotbox +Xaldon_WebSpider Xenu -YoudaoBot -Zade -arquivo-web-crawler +Zermelo +Zeus +ZumBot +ZyBorg +archive.org_bot autoemailspider backlink-check -check1.exe -coccocbot-web cognitiveseo -evc-batch -gopher +crawl.sogou.com +crawler.feedback heritrix -instabid -ips-agent -lwp-trivial +internetVista monitor +muhstik-scan netEstate NE Crawler -page scorer plumanalytics +probe-image-size +ripz +satoristudio.net +scalaj-http scan.lol serpstatbot sexsearcher sitechecker.pro -sp_auditbot spyfu sysscan -tAkeOut +trendiction.de ubermetrics-technologies.com -webmeup-crawler -zgrab +x09Mozilla +xpymep1.exe diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 5da5c87fe..e03b04a42 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -404checker +01h4x.com 404enemy -Abonti +80legs +AIBOT +ASPSeek Aboundex -AhrefsBot -Alexibot -AllSubmitter +Aboundexbot +AiHitBot +Alligator Anarchie -Apexoo -Aspiegel -AwarioSmartBot +Anarchy +Asterias +AwarioRssBot +BackDoorBot BackStreet +Backlink-Ceck +Bandit Barkrowler -Battleztar Bazinga -BetaBot -Bitacle -Blackboard -BlowFish +Bigfoot +Black Hole Boardreader Bolt +Brandprotect Buck +BuiltBotTough Bullseye -BuzzSumo -CazoodleBot -CensysInspect -CheeseBot +CATExplorador +CSHttp +Calculon CherryPicker -Cliqzbot +Cloud mapping +Cocolyzebot Cogentbot -Collector Copyscape -Custo -DIIbot -DataCha0s -DatabaseDriverMysqli -Dirbuster -Discobot +Cosmos +Craftbot +CrunchBot +Deusu +Digincore Discoverybot -DittoSpyder -DomainCrawler -DomainStatsBot -Drip -Express WebPictures +Dispatch +Dotbot +Dragonfly +EMail Siphon +EMail Wolf +Ebingbong +Ecxi +Evil +Exabot ExtLinksBot +Extractor ExtractorPro -Extreme Picture Finder EyeNetIE -FHscan -FemtosearchBot Firefox/7.0 FlashGet -FyberSpider -Fyrebot -G-i-g-a-b-o-t -GermCrawler -GetRight -Gigabot +Flunky +Fuzz +GT::WWW Go!Zilla -Grabber -Grafula +Go-Ahead-Got-It +GrabNet GrapeFX +GrapeshotCrawler HEADMasterSEO HTMLparser HTTP::Lite -HTTrack -Haansoft Havij Heritrix -HybridBot +Humanlinks IDBTE4M -IRLbot +IDBot Id-search IlseBot -InfoTekies +Image Fetch +Image Sucker Intelliseek InterGET -InternetSeer -IstellaBot +Iskanie +JOC Web Spider +Jbrofuzz JetCar Jetty JikeSpider +Jyxobot Keyword Density -Kozmosbot -LNSpiderguy -Lanshanbot Larbin -LeechGet +Leap +LexiBot +LibWeb LieBaoFast Lightspeedsystems -Likse LinkScan +LinkWalker +LinkextractorPro LinkpadBot -LinqiaMetadataDownloaderBot -LinqiaScrapeBot Litemage_walker -MIIxpc -MSIECrawler +Ltx71 +MIDown tool +MQQBrowser +MTRobot Magnet -Mail.RU_Bot +Majestic SEO Majestic-SEO -MarkWatch -Mass Downloader -Mata Hari -MauiBot +Mb2345Browser MeanPath Bot Meanpathbot -Mediatoolkitbot MegaIndex.ru -Msrabot +MicroMessenger +Microsoft Data Access +Moblie Safari +Mojeek +Mozlila +Mr.4x3 Musobot -Name Intelligence Nameprotect +Needle Net Vampire NetAnts -NetZIP +NetLyzer Netcraft +Netvibes +NextGenSearchBot Niki-bot Nikto -Nmap Nuclei Nutch -Offline Explorer -Offline Navigator -OpenVAS +Octopus +OnCrawl Openvas -OrangeSpider -OutclicksBot -PECL::HTTP -PHPCrawl -PageThing.com +OrangeBot +OutfoxBot +POE-Component-Client-HTTP +Page Analyzer Panscient Papa Foto -Petalbot +Pavuk Picscout Picsearch -PictureFinder -Pimonster -Pixray PleaseCrawl ProPowerBot ProWebWalker +Probethenet +Pu_iN Pump PxBroker -Quick-Crawler -RankActiveLinkBot -Rankivabot +PyCurl +QueryN Metasearch +RankFlex +RankingBot2 RankurBot Re-re -RealDownload -Reaper +Recorder RedesScrapy -RepoMonkey -RocketCrawler -SEOlyticsCrawler +Rogerbot +SBIder +SEOkicks SEOstats -SMTBot +SalesIntelligent Scanbot +ScoutJet Scrapy -SearchmetricsBot +ScrepyBot +Semrush +SemrushBot +SeoSiteCheckup +Seomoz Siphon SiteExplorer -SiteLockSpider -Sitebeam -Siteimprove -SmartDownload -Snoopy +SiteSucker +Snake +Snapbot SocialRankIOBot Sociscraper -Sosospider -Sottopop +Sogou web spider SpaceBison -Spammen -SputnikBot -Sqlworm -Sqworm -Steeler +Spbot Sucker -Sucuri -SuperHTTP -Suzuran -Telesoft +Surfbot +Swiftbot +T0PHackTeam +T8Abot +Telesphoreo Telesphorep +TightTwatBot Toata -Tracemyfile -Trendiction -TurnitinBot +True_Robot +Turnitin +URLy.Warning Upflow +V-BOT +VCI +Vacuum +Vagabondo VelenPublicWebCrawler +VeriCiteCrawler +VidibleScraper Virusdie -WASALive-Bot +WBSearchBot WEBDAV WISENutbot +WWW-Collector-E WWW-Mechanize -WWW::Mechanize -WWWOFFLE Wallpapers/3.0 -WallpapersHD -WeSEE +Web Auto Web Collage -Web Fetch Web Fuck +Web Pix Web Sauger +Web Sucker WebAuto -WebGo IS -WebLeacher -WebPix -WebSauger -WebWhacker -Website Quester -Whack -Whacker +WebBandit +WebCollage +WebCopier +WebFetch +WebStripper +WebSucker +WebZIP +Webshag +WebsiteExtractor +WebsiteQuester Whatweb -Who.is Bot Widow +WinHTTrack +WiseGuys Robot Wonderbot -Wotbox -Wprecon -Xaldon_WebSpider +YoudaoBot Zauba +ZoominfoBot ZyBorg -arquivo-web-crawler +archive.org_bot arquivo.pt autoemailspider -backlink-check check1.exe crawler.feedback crawler4j -demandbase-bot +eCatch facebookscraper -heritrix -ips-agent isitwp.com -lwp-request lwp-trivial -meanpathbot -oBot -page scorer -pcBrowser -polaris version +magpie-crawler probe-image-size s1z.ru -scan.lol -seobility -sexsearcher +serpstatbot sitechecker.pro siteripz sogouspider +sysscan tAkeOut -xpymep1.exe -zauba.io +ubermetrics-technologies.com +voyagerx.com diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 664fc44ac..f27805d2e 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -ahrntal.verymes.xyz -angel1777.kz -dosugrostov.site -everypony.ru -fuckingawesome.com -new-apps.ru -npoet.ru -shopperifymac.com -siteaero.com -soheavyblog.com +adloads.net +camdenmemorials.com +casinopinup-wins.com +crd.clan.su +extads.net +healthcarestore.info +humanelydrew.com +kraljeva-sutjeska.com +seoprofiler.com +toondinsey.com diff --git a/dev-tools/test_units/random-referrers-for-test.list b/dev-tools/test_units/random-referrers-for-test.list index 2a55fc719..6196f8a46 100644 --- a/dev-tools/test_units/random-referrers-for-test.list +++ b/dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -0lovespells0.blogspot.com -12masterov.com -1314dh.com -14b.info -2015god.org -256bit.by -2ads.co.uk -55wmz.ru -5elementov.ru -72-news.com -8kisses.eu -9icmzvn6.website -a14download.com -aa8780bb28a1de4eb5bff33c28a218a930.com -aanapa.ru -abusora.com -adanih.com -adclickservice.com +00webcams.com +101billion.com +3rbseyes.com +aa8b68101d388c446389283820863176e7.com +abtasty.com +adf.ly +adloads.com adnotbad.com -ads-cool.pro -ads.gold -adzerg.com -aerogo.com -aktobe.xkaz.org -alarmobninsk.ru -alborzan.com -albumsuper.info -alert-hgd.xyz -alfa9.com -alienwheels.de -aliexpress.com -alkoravto.ru -all4wap.ru -allnews.md -allstatesugarbowl.org -ameblo.jp -ananumous.ru +adpremium.org +adsloads.com +adtology.com +adventureparkcostarica.com +aero2.ru +afftrack001.com +afslankpillen2017nl.eu +ahrntal.verymes.xyz +alif-ba-ta.com +allmarketsnewdayli.gdn +alltheviews.com +alphacarolinas.com +alphaforum.ru +alt-servis.ru +amt-k.ru +anal-acrobats.com anapa-inns.ru -angkortours.vn -animalia-life.club -anjalika.co.in -apiitsthirteende-a.akamaihd.net -apispringsmartne-a.akamaihd.net -appartement-stumm.at -arendatora.ru +angigreene.com +anime.dougasouko.com +animebox.com.ua +app-ready.xyz +approved.su +approvedlocal.co.za +aquarium-pro.ru +arabsexxxtube.com +arendovalka.xyz arkartex.ru -arquivo.pt -artel116.ru -asiengirls.net -asociatia-tipografilor-transilvania.ro -atmagroup.ru -atofilms.com -auto4style.ru -autonew.biz -autoplate.info -ayerbo.xhost.ro -babyfactory.fr -backlinkwatch.com -baditri.com -bagcionderlab.com -bangla-chat-uk-paralud.ga -barbourjackets.online -bard-real.com.ua -begalka.xyz -beget.tech -belinvestdom.by -bestofferswalkmydogouteveryday.gq -bestwaystogetpaid.us -bestwebsiteawards.com -big-boards.info -bigpictures.club -blackcurranthumidifiers.site -blackle.com -blogporn.in -bluerobot.info -bonux.nextview.ru -bookmaker-bet.com -bookmark4you.com -bouda.kvalitne.cz -brandbucket.com -brendbutik.ru -brothers-smaller.ru +arkkivoltti.net +artpicso.com +asacopaco.tk +atovh.local-finders.com +auto-moto-elektronika.cz +auto-zapchasti.org +availit.weebly.com +avtolombard-krasnodar.com +baladur.ru +bdsmgalls.net +beachtoday.ru +beauty-b0x.pl +beclean-nn.ru +bestmarriages.com +bestssaker.com +bet365.com +betonka.pro +bezsporno.ru +bildsuche.ru +bio-japan.net +blobar.org +blockworld.ru +blog.koorg.ru +boc.kz +bolegapakistan.com +brandehk.dk +brandimensions.com +bscodecs.com btnativenav.com +buckspar.gq +builtwith.com +bumascloset.com bumskontakte.org -buttons-for-your-website.com -buyclomidonlaine.com -buyhoverboard.com -buzzzg.men -cakemediahq.com.au -calc-for-credit.ru +cablecar.us +calstaterealty.us camdolls.com carloans.com -cartierwatch.xyz -cbb1smartlist12.click -cete.ru +carson.getenjoyment.net +cartierloveringreplica.xyz +ccbill.com +ccjp.eu +cdnnetwok.xyz +cfcl.co.uk +chatroulette.video chatroulette.world -cheapjerseysfootballshop.com -cherubinimobili.it -ci.ua -cinemaenergy-hd.ru +chinajerseyswholesalecoupons.com +ciekawinki.pl +cinemaflix.website citizenclsdriveri7.pen.io -clash-clans.ru -click2pawn.com -clippingphotoindia.com +clayvasedesigns.tk +cleanallspyware.com +clickintext.net cocolyze.com -comissionka.net -compliance-fred.top -confib.ifmo.ru -contentsexpress.com -cookie-law-enforcement-ff.xyz -coolgramgoods.com -copenergo.ru -countbertwistdisp26.soup.io -country-chic.ru -crackguru.tk -crazyprotocol.com -creativehutindia.com -crest3d.ru -custodieva.ru -customchocolate.business-for-home.com -customcollegeessays.net -czat.wp.pl -dafatiri.com +colehaanoutlet.store +compliance-alex.top +compliance-alexa.top +compliance-willy.top +cpabegins.ru +cracksplanet.com +d-black.bz +daisye.top dangphoto.trade -datadepths.com -deanmoore.ie -decorationspcs.com -defenderxtactical.com -derevesendeco.com -dev.citetick.com -doeco.ru -dolohen.com -domaincheck.io -domaincrawler.com -dotmass.top -doublepimp.com -download-of-the-warez.blogspot.com -drupa.com +decoratinghomes.ga +deda-moroza-zakaz.ru +deplim.com +design-ideas.info +designdevise.com +dgfitness.co +diesel-parts28.ru +diy-handmade-ideas.com +docsportal.net +downloadkakaotalk.com +downvids.net +doxyporno.com dustyorate.com -ed-shop01.ru -editmedios.com -edudocs.net -elektrischezigaretten2.beeplog.com -elektronischezi.livejournal.com -elektrozigaretten1.webs.com -eloconcream.blogspot.com -emerson-rus.ru -en.home-task.com -englishdictionaryfree.com -enthuse.computernetworksonline.com -eorogo.top -erotikchat-24.com -es5.com -evehemming.blogspot.com.au -exchangeit.gq -exdocsfiles.com -expertblog.info -eyelike.com.ua -ez8motelseaworldsandiego.com -ezigarettenshop1.yolasite.com -f07.de -falcon-images.blogspot.com -farmingworm.com +dvr.biz.ua +e2click.com +eaplay.ru +ecookna.com.ua +eduserver.net +electrik-avenue.com +electronicadirect.com +elektrischezi.canalblog.com +elektrozigarette2.wordpressy.pl +enews.tech +engines-usa.com +epicbrogaming.com +estelight.ru +everflora.ru +expertnaya-ocenka.ru +exportshop.us +extener.org +ezigarettenkaufen2.dreamwidth.org +famix.xyz +fashionavenuegame.com +fba-mexico.com +fbdownloader.com +fdzone.org feel-planet.com -fefo.gdn -felizporno.com -fetroshok.ru -figuringmoneyout.com -finansov.info -findpics.pw +femdom.twiclub.in +fenoyl.batcave.net +fetishinside.com +fickenbumsen.net +findanysex.com findthe.pet -foodcrafts.website -fotoxxxru.com -franch.info -front.to -gallerylisting.com -gamesprite.me -gamewrath.com -gdzkurokam.ru -getoutofdebtfree.org -gheus.altervista.org -gigablast.com -globalscam.ga -gne8.com -go2album.com -google-liar.ru -googleseo.com.tr -graphics8.info -greatdealshop.com -greatzip.com -gsasearchengineranker.xyz -gsasearchenginerankerdiscount.com -gta-top.ru -guidefs.ru +fireads.men +firesub.pl +fishingwholesale.us +flex4launch.ru +florida-tourism.net +forensicpsychiatry.ru +forum.tvmir.org +freakycheats.com +fun2cell.net +galeria-zdjec.com +gate5.co.za +gembird.com +germetiki.com.ua +getlamborghini.ga +ghazel.ru +gigixo.com +gloverid.site +googlepositions.com +gopixdatabase.com +gratuitbaise.com +gsasearchengineranker.pw +guardlink.org +hackers-crackers.tk hahashka.ru -hamptonoaks.ca -handicapvantoday.com -hard-porn.mobi -hawaiielectriclight.com hazardky.net -hermesreplica.pw -herokuapp.com -hiwibyh.bugs3.com -hmywwogw.bloger.index.hr -hothor.se -howlongdoestizanidinestayinyoursystem.blogspot.com -howmuchdoestizanidinecost.blogspot.com +hdpixent.com +helicalpile.us +help.tpu.ru +hledejvshopech.cz +hoholikik.club +hollywoodactress.info +home.myplaycity.com +homedecoguide.info +hotgirlhdwallpaper.com +hotkeys.com +houseofgaga.ru hyipmanager.in -idealtits.net -images-free.net -imgarit.pw -impisr.ru -innodgfdriverhm.aircus.com -instabid.tech -intermesh.net -invest-pamm.ru -iqoption.com -iqupdatetmz.win -istmira.ru -itservicesthatworkforyou.com -j-times.ru -jaxcube.info -jerseyssportsshop.com -juliaworld.net -kandidos.com -karaganda.xkaz.org -kartiniresto.com -keenoutlet.online -keywesthideaways.co -keywordblocks.com -kidd.reunionwatch.com +i4track.net +indetiske.ya.ru +infostatsvc.com +innoslicon.com +instakink.com +intelhdgraphicsgtdrive6w.metroblog.com +intervsem.ru +investyb.com +io9.com +iqoption-bin.com +isoveti.ru +iyasimasennka.com +jacago.com +jav-fetish.site +javspace.net +jmat.cn +johannesburgsingles.co.za +johnnyhaley.top +jrcigars.com +juliadiets.com +kemerovo.zrus.org +kevblog.top +kicknights.gq +kinky-fetishes.com kinoduh.ru -kirov.zrus.org -kiwi237au.tk -kletkimehan.ru +kinofak.net klumba55.ru -knowyournextmove.com -kostenlos-sexvideos.com -kovrenok.ru -kraljeva-sutjeska.com -krasnodar.zrus.org -kurbappeal.info -kursy-ege.ru -lacapilla.info +komputernaya-pomosh-moscow.ru +konteiner24.com +kostenloser-sex.com +krasivye-devushki.net +kustanay.kz +lafriore.ru +languagecode.com lapolis.it -lasvegaslockandsafe.com -lazy-z.com -lenpipet.ru -libertybilisim.com -lida-ru.com -linkredirect.biz +lazymanyoga.com +lernur.net +lightinghomes.net lion.conboy.us -livejasmin.com -liver-chintai.org -lodki-pvh.dp.ua -lottospring.com -lovasszovetseg.hu -luckyshop.net.ua -m3gadownload.pl -macnewtech.com -macrotek.ru -mailemedicinals.com -maridan.com.ua -marmitaco.cat -mebel-vstroika.ru -medicovi.com -medmajor.ru -meds-online24.com -mensandals.xyz -mexicosleevegastrectomy.com -mini.7zap.com -mirzonru.net -mncrftpcs.com -mobicover.com.ua -mobile-appster.ru -mojaocena.com +lirunet.ru +liveinternet.ru +livingroomdecor.info +ljusihus.se +locksmith.jp +lolnex.us +lonely-mature.com +lostfilm-online.ru +lowephotos.info +m5home.ru +mafcards.ru +marinetraffic.com +marmotstore.online +marwer.info +mcadamssupplyco.com +mediaoffers.click +mediawhirl.net +medkritika.ru +megaindex.ru +metabar.ru +meteocast.net +meteostate.com +midst.eu +milblueprint.com +millionare.com +mir-limuzinov.ru +mirobuvi.com.ua +misandesign.se +mixed-wrestling.ru +mobilierland.com +moesen-ficken.com mompussy.net -monsterdivx.com -monsterdivx.tv -mrbojikobi4.biz -mrwhite.biz -mulberryoutletonlineeu.com -mundoaberrante.com -muznachas-service.ru -mxgetcode.com -mypornfree.ru -narosty.com -netvouz.com -networkad.net -newtechspb.ru -nhl17coins.exblog.jp -niki-mlt.ru -nikitsyringedrivelg.pen.io -novgorod.xrus.org -nzfilecloud.weebly.com -odywpjtw.bloger.index.hr -offer.camp -ok-ua.info -oksrv.com -onlinefilmz.net -onlineporno.site -ontargetseo.us -ooo-olni.ru -openfrost.net -outrageousdeal-a.akamaihd.net -oynat.info -panamaforbeginners.com -partner-cdn.men -partnerads.men -parvezmia.xyz -pb-dv.ru -phelissota.xyz -phormchina.com -photochki.com -pinsdaddy.com -pixgood.com -pkr1hand.com -plastgran.com -pontiacsolstice.info -popmarker.com -poppen-nw.net -popugaychiki.com -porn555.com +motherboard.vice.com +mototsikl.org +mp3films.ru +mrmoneymustache.com +mrvideospornogratis.xxx +muabancantho.info +musezone.ru +music.utrolive.ru +musicstock.me +my-aladin.com +mycouponizemac.com +myhitmp3.club +mylovelibrabry.com +mypets.by +myplaycity.com +nailsimg.com +nakozhe.com +nakrutka.cc +narkologiya-voronezh.ru +nastydollars.com +naturalpharm.com.ua +nerudlogistik.ru +netfacet.net +newstudio.tv +nibbler.silktide.com +nikhilbahl.com +notasprensa.info +nufaq.com +offer.wpsecurity.website +office2web.com +okmusic.jp +olvanto.ru +omgtnc.com +onefilms.net +onlainbesplatno.ru +online-podarki.com +ooo-gotovie.ru +oops-cinema.ru +openlibrary.org +optikremont.ru +orenburg-gsm.ru +orgasmatrix.com +owathemes.com +oxford-book.com.ua +palocco.it +pandroid.co +par-fallen.ga +partner-host.men +partnersafe.men +partybunny.ru +pateaswing.com +pc-test.net +pdamods.ru +pdn-4.com +perso.wanadoo.es +photosaga.info +phrcialiled.com +picphotos.net +pierrehardysale.online +pijoto.net +pipki.r.acdnpro.com +piulatte.cz +pizdeishn.com +pliks.pl +poffet.net +popup-jdh.xyz +porn-w.org pornhubforum.tk -pornonik.com -pornoslive.net -pornotubs.com -pornozhara.com -portside.cc -pozdrawleniya.com -preconnubial.usuby.site -prefersurvey.net -preventheadacheguide.info -privatefx-in.ru -prize44.com -profolan.pl -prointer.net.ua -prom23.ru -prostitutki-belgoroda.org -prostitutki-novgoroda.org -purplestats.com -putevka24.ru -putitin.me -py100.ru -pyramidlitho.webs.com +pornofiljmi.com +pornogad.com +pornogratisdiario.com +portadd.men +producm.ru +produkto.net +projectforte.ru +promover.org +prospekt-st.ru +prostitutki-rostova.org +pshare.biz +pskcijdc.bloger.index.hr +pukaporn.com +purplesphere.in +pussyfleet.com +qor360.com quelle.ru -radiogambling.com -randalljhoward.com -rangjued.com -rasteniya-vs-zombi.ru -redbottomheels.xyz -registry-cleaner.net -rentaremotecomputer.com -rfserial.net -romhacking.ru -sale-japan.com -sampleletters.net -saugatuck.com -sax-sex.com -scanner-john.top -scanner-josh.top -scanner-olga.top -scanner-willy.top -scenarii-1-sentyabrya.uroki.org.ua -seansonline24.pl -search.1and1.com -searchtooknow-a.akamaihd.net -selectads.men -seo-prof1.xyz -seofied.com -seoprofiler.com -seorussian.ru -seotoolsagency.com -sergiorossistore.online -serving.adbetclickin.pink -servisural.ru -seryeznie-znakomstva.ru -sevendays.com.ua -sexsearch.com +quit-smoking.ga +rada.ru +ragecash.com +rapidgator-porn.ga +raschtextil.com.ua +rating-casino2021.ru +realtytimes.com +recordpage-a.akamaihd.net +registratciya-v-moskve.ru +renecaovilla.online +rethinkwasteni.info +reversing.cc +rezeptiblud.ru +rootandroid.org +rostov.xrus.org +running-line.ru +runofilms.ru +rus-pornuha.com +ruscoininvest.company +rvtv.ru +sarafangel.ru +scanner-andrew.top +scanner-julianna.top +scanner-marwin.top +scansafe.net +scoopquest.com +screentoolkit.com +searchimpression.com +seeingmeerkat.com +seemoreresultshu-a.akamaihd.net +seeresultshub-a.akamaihd.net +seokicks.de +sex-tracker.com +sex-tracker.de +sexkrasivo.net +sexvideo-sex.com +sexy.babes.frontend-stack.top sexyebonyteen.com -sexytrend.ru -shakhtar-doneck.ru -shiksabd.com +sexyteens.hol.es +shariki-zuma-lines.ru +shoesonlinebuy.cn +shopfishing.com.ua +sibdevice.ru siha.de -simply.net -sims-sims.ru -site-analyzer.com -site-auditor.online -sitebeam.net -sitopreferito.it -sivs.ru -skincrate.net -sky-mine.ru -sluganarodu.ru -smailik.org -small-games.biz -snip.tw +site-speed-check.site +site5.com +slkrm.ru +smallseotools.com +smstraf.ru +social-s-ggg.xyz +socialbuttons.xyz socialtrade.biz -sockshare.net -softomix.com -soviet-portal.do.am -speechfoodie.com -sprttrack.com +solmarket.by +spamnuker.com +spasswelt.xyz +spidtest.space spywareit.com -srgwebmail.nl -stairliftstrue.com -steebook.com -sterva.cc -streetfooduncovered.com -stroimajor.ru -stroyhelp-dv.ru -sudexpert66.ru -sukarame.net -supers.com.ua -supervesti.ru -survival.betteroffers.review +squidoo.com +startwp.org +steame.ru +stefanbakosab.se +stockquotes.wooeb.com +strana-krasoty.ru +stuff-about-money.com +sucsesofinspiration.com +sunflowerdrawingpaintings.blogspot.com +supermesta.ru suture.co -svensk-poesi.com -sygraem.com -tastyfoodideas.com -tech4master.com +svarkagid.com +svtrd.com +taaaak.com +takethatad.com +taqywu51.soup.io +tattooreligion.ru +tedxrj.com teguh.info -tehngr.ru -thecounter.com -thedownloadfreeonlinegames.blogspot.com +tetracsaudi.com +tgtclick.com +thedownloadfromwarez.blogspot.com themeforest.net -thin.me.pn +timdreby.com +time-japan.ru tivolibasket.it -tizanidinedrugtest.blogspot.com -tizanidinevscyclobenzaprine.blogspot.com -tn811.us -tnctrx.com -top-deal.com.pl -top-karkas.ru -topads.men -torture.ml -tourcroatia.co.uk -tpu.ru -tracfone.com +tizanidinehcl4mgtablet.blogspot.com +tizanidinenarcotic.blogspot.com +tizanidinerestlesslegsyndrome.blogspot.com +tizanidinestreetprice.blogspot.com +tizanidineusedfor.blogspot.com +top-study.work +top1-seo-service.com +top10-online-games.com track2.shop -tradedeals.biz -traffpartners.com -ttsq.fr -tube8.com -tversvet.ru -typimga.pw -ucsol.ru -ukkelberg.no +traderzplanet.in +tramadolandtizanidine.blogspot.com +traxdom.ru +trion.od.ua +triplepanda.xyz +tsyndicate.com +turbo-suslik.org +tvnewsclips.info +tvory.predmety.in.ua +twiclub.in +uchetunet.su ultimateclassicrock.com -url2image.com -urlopener.com -utiblog.fr -vadimkravtcov.ru -valmetrundan.se -vapomnoncri.tk -vbabule.net -vedomstvo.net -vektorpress.ru -videochat.cafe -videonsk.com -viteonlusarezzo.it -voloomoney.com -vostoktrade.info -vykupavto-krasnodar.ru -waterefficiency.co -waycash.net -web-betting.ru -webenlace.com.ar +unionmarkt.de +uptime-alpha.net +upupa.net +urll.eu +uroffer.link +us-america.ru +versaut.xxx-cam.webcam +video-production.com +vietimgy.pw +vip2ch.com +viven.host.sk +voluumtrk.com +vozbujdenie.com +vpnmouse.com +vzlom-na-zakaz.com +w3data.co +walpaperlist.com weblo.com -website-stealer.nufaq.com -whatistizanidine2mg.blogspot.com -white-truck.life -whiteelephantwellington.com -wikes.20fr.com +webs.com +welcomeauto.ru +wfb.hatedriveapart.com +whatsappdownload10.com +whos.amung.us +wingsoffury2.com winner7777.net -wordkeyhelper.com -writersgroup580.web.fc2.com -www.888.com -x5market.ru -xchangetrak.com -xn----7sbaaabaei0cc8aj5bj0bncejx.xn--p1ai -xn----itbeirbjbi7bc6bh2d.xn--p1ai -xn--80aanaardaperhcem4a6i.com +witclub.info +woman-tampon.ru +womensplay.net +worldinternetauthority.com +wsgames.ru +x-diesel.info +xblog.in +xdoza.com +xfire.com +xitjw.info +xlovecam.com +xn-----8kcatubaocd1bneepefojs1h2e.xn--p1ai +xn--80aaafbn2bc2ahdfrfkln6l.xn--p1ai +xn--80aaaks3bbhabgbigamdr2h.xn--p1ai +xn--80aagddcgkbcqbad7amllnejg6dya.xn--p1ai +xn--80aeahghtf8ac5i.xn--p1ai +xn--80aikhbrhr.net +xn--h1aakne2ba.xn--p1ai +xxxdatinglocal.us yaderenergy.ru -youghbould.wordpress.com -yoursite.com -youtoner.it -youtubedownload.org -yurgorod.ru -zahvat.ru -zaidia.xhost.ro -zakazfutbolki.com -zakon-ob-obrazovanii.ru -zakonobosago.ru -zeikopay.com -zerocash.msk.ru +you-shall-not-pass.is74.ru +yougotanewdomain.com +your-bearings.com +yourmovies.pl +z-master.ru +zagadki.in.ua +zatjmuzu.info zeroredirect.com -zeroredirect5.com -zigarettenonl.canalblog.com -znakom.sibtest.ru +zeroredirect11.com +zeroredirect6.com +zhuravlev.info +zigarettenonlinekaufen1.bloog.pl +zigarettenonlinekaufen1.blox.pl +zirondelli.it +znakomstva-moskva77.ru +zoolubimets.ru +zophim.me +zscalertwo.net diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index cf031575d..c02972932 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -00webcams.com -1008.su -123locker.com -12u.info -3-letter-domains.net -51unlim.ru -66cpwgln.space -adinterax.com -adultwebhosting.info -advertisingtag.net -agahinameh.com -agreda.pluto.ro -allboard.xobor.de -alta-realestate.com -altamayoztourism.com -an-donut.com -apartment.ru -apelsinnik.website -atdedinotuho.tk -atofilms.com -audiobangout.com -autoblogger24.info -ayanaspa.com -azartniy-bonus.com -bankmib.ru -beauty-bracelet.com -beetpics.pw -bestcoin.cc +00author.com +1adult.com +24x7-server-support.site +2drittel.de +2pxg8bcf.top +300richmond.co.nz +aa08daf7e13b6345e09e92f771507fa5f4.com +adbetclickin.pink +addray.pro +adprotect.net +affiliate-fr.com +alert-fjg.xyz +alveris.ru +anaseracresar.tk +anjalika.co.in +apibrowsesmartne-a.akamaihd.net +app5.letmacworkfaster.world +apptester.tk +as5000.com +asrv-a.akamaihd.net +ayodhya.co +bag77.ru +baksman.com +baladur.ru +balitouroffice.com +beget.tech +best-way.men besthoro.ru -bestmouthwash.club -bestofpicture.com -betterhdporn.com -beyan.host.sk -bezcmexa.ru -bikini-image.com -bimatoprost-careprost.com -boanonihaca.tk -bookmark4you.com.biz -bpro1.top -break-the-chains.com -buzzzg.men -cakesplus.com.au -cartechnic.ru -cementaresearch.se -certifywebsite.win -cfacarrosserie74.com -cheapcigarettesc.info -chuckguilford.com -cialisovercounteratwalmartusa.com -cigarpass.com -clicksor.com -cntravelre.com -cometorussia.net -compliance-andrew.top -compliance-jack.top -consorzioilmosaico.org -contentlook.co -cosmeticswomens-womensports.rhcloud.com -davebestdeals.com +bestserials.com +bestsexyblog.com +besttorrentknifta.weebly.com +blog.remote-computer.de +blogs.rediff.com +bocoarchives.org +bookhome.info +brainboosting.club +brainboostingsupplements.org +bravegirlsclub.com +bristolhotel.com +bryansk.zrus.org +buckspar.gq +bumble.cheapwebsitehoster.com +businescoop.men +buzzurl.jp +camdenmemorials.com +camdolls.com +carloans.com +censys.io +cheapjerseysap.com +children-learningreading.info +chocoslim.pro +cicaki.net +cinemaenergy-hd.ru +clarithromycin500mg.com +compliance-julianna.top +contextualyield.com +coslab.club +cosmetic.donna7753191.ru dbmkfhqk.bloger.index.hr -diminishedvalueoforegon.com -dirtpics.pw -documentsite.net -dostavimvdom.ru -drugstoreforyou.com -ebonyporn.site -ebooknovel.club -elektrozigarette2.webs.com -elektrozigaretten2.yn.lt -essayassist.com -europages.com.ru -ezigarettekaufen2.blox.pl -familyholiday.ml -faptitans.com -findthe.pet -flytourisme.org -forum20.smailik.org -foxweber.com -free-fb-traffic.com -free-gluten.ru -freelifetimefuckbook.com -frequiry.com -freshdz.com -froggytube.com -gamedayassist.com -ge0ip.com -goodwinmetals.co -gorodservis.ru -gpms.org.my -greenzaim.ru -gta-top.ru +decorazilla.com +demon-tweeks.com +dengi-pod-zalog-nedvizhimosti.ru +derevesendeco.com +digitalfaq.com +directivepub.com +disruptingdinnerparties.com +distver.ru +domashneeruporno.com +dorratex.tn +dotmass.top +eaplay.ru +earthmagic.info +eduinfosite.com +elektrozigaretten1.tumblr.com +elitepcgames.com +emerson-rus.ru +encodable.com +epicbrogaming.com +ereko.ru +estelight.ru +evehemming.blogspot.com.au +extstat.com +ezigarettenshop2.myblog.de +falllow.gq +felizporno.com +filefilter.weebly.com +fiverr.com +flowersbazar.com +forumprofi.de +fuckbuddybestgilf.info +fusoradio.info +fyl.com.ru +game300.ru +gamezblox.com +getpopunder.com +gsasearchengineranker.xyz +gyffu.com +hcate.com hdpixion.com -help.tpu.ru -heroz.fr -hmmm.cz -ie.57883.net -igrovyeavtomaty777.ru -ilo134uloh.com -imfamous.info -infogame.name -integritylandscapeservices.com -internet-apteka.ru -intervsem.ru -iqupdatetmz.win -itis4you.com -jackwolfskinoutlet.online -janerikholst.se -janettabridal.com +heartofbeijing.blogspot.com +herokuapp.com +hobbyhuren-datenbank.com +home-task.com +homegardenlova.com +hornymatches.com +hoverboard360.es +hut1.ru +hvd-store.com +imgarcade.com +inboundlinks.win +inclk.com +infobabki.ru +insta-add.pro +internetproviderstucson.com +islandminingsupply.wordpress.com +izhevsk.xrus.org jetsli.de -jyrxd.com -karadene.com -kartiniresto.com -keywordglobal.co.za -keywordsking.com -khadastoafarde.tk -lafourmiliaire.com -ldrtrack.com -leadwayau.com -linerdrilling.com -lookingglassemb.com -louboutinreplica.xyz -luchshie-topcasino.ru -macrotek.ru -mainlinehobby.net -manimpotence.com -mantramusic.ru -max-p.men -med-zdorovie.com.ua -mediaoffers.click -medictube.ru -medmajor.ru -metascephe.com -migente.com -mmgq.ru -mobicover.com.ua -mobile-appster.ru -moi-glazki.ru -monsterdivx.com -montazhnic.ru -moonci.ru -mug-na-chas-moscow.ru -myxdate.info -na-telefon.biz -napalm51.nut.cc -nicovideo.jp -norththeface.store -notasprensa.info -nudejapan.net -nudo.ca +jovencitas.gratis +kino2018.club +kiskinhouse.com +konpax.com +kosova.de +kretpicf.pw +kurbappeal.info +landmania.ru +legalrc.biz +letmacworkfaster.site +lisque.batcave.net +liverpool.gsofootball.com +love-baby.cz +lrac.info +mackeeper-center.club +magazintiande.ru +matsdale.com +max-eclat.men +mebel-vstroika.ru +medicines-choice.com +microsoftportal.net +mobilierland.com +moetomnenie.com +moncleronline.xyz +muschisexbilder.com +muztops.ru +mydirtyhobby.com +mypets.by +mypornfree.ru +netoil.no +netpics.org +nlfjjunb5.ru +oakridgemo.com +offer.camp +offer.wpsecurity.website office2web.com -offtopic.biz -oivcvx.website -omoikiri-japan.ru -onstrapon.purplesphere.in -oops-cinema.ru -openfrost.net -orion-code-access.net -owathemes.com -p-business.ru -panouri-solare-acoperis.com -patterntrader-en.com +olvanto.ru +partner-host.men +pewit.pw photochki.com -pk-services.ru -plastjulgranar.se -poem-paying.gq -polytopesexempt.com -porno-transsexuals.ru -pornokorol.com -pornozhara.com -potoideas.us -princevc.com -prointer.net.ua -qpypcx.com -quebec-bin.com -randki-sex.com +photosaga.info +physfunc.ru +picturesfrom.com +pinsdaddy.com +poffet.net +popads.net +pornophoto.xyz +portside.xyz +pozdravleniya-c.ru +predmety.in.ua +psvita.ru +purplestats.com +pussyspace.net +qualitymarketzone.com +r-e-f-e-r-e-r.com +razamicroelectronics.com recipedays.ru -respublica-otel.ru -rfserial.net -riders.ro -roleforum.ru -royal-betting.net -rucrypt.com -rumamba.com -rvtv.ru -saddiechoua.com -saitevpatorii.com -savetubevideo.com -scanner-nelson.top -seomarketings.online -serving.adbetclickin.pink -sethrollins.net -sexkrasivo.net -shakhtar-doneck.ru -shariki-zuma-lines.ru -sheerseo.com -ship-marvel.co.ua -silvermature.net -simple-image.com.ua -slavic-magic.ru -social-s-iii.xyz -soft1.ru -startwp.org -stmassage.ru -supermodni.com.ua -swagbucks.com -telegraf.by -telsis.com -todohr.com -tomck.com -top10registrycleaners.com -track-rankings.online -trubywriting.com -tube8.com +redbottomheels.xyz +refads.pro +reimageplus.com +rentehno.ru +replicaclub.ru +romhacking.ru +rp9.ru +sarm.tk +savememoney.co.za +sawin.beth.webd.pl +sbornik-zakonov.ru +seadragonherbery.com +searchinquire.com +seeingmeerkat.com +semprofile.com +semrush.com +seojokes.net +sexreliz.com +sexvideo-sex.com +shoppingjequiti.com.br +shoppingmiracles.co.uk +shopwme.ru +sitebeam.net +siteonomy.com +slavkokacunko.de +sloopyjoes.com +sneakyboy.com +snip.tw +soc-econom-problems.ru +socialsignals24.com +soft-program.com +sport-video-obzor.ru +stauga.altervista.org +stpicks.com +stroimajor.ru +stroydetali.ru +sugarlyflex.pw +sugvant.ru +taboola.com +taxi-v-eisk.ru +thin.me.pn +toonfamilies.net +touchmods.fr +trafficfactory.biz +ttsq.fr tubeline.biz -tubo360.com -tx41tclega.ru -udayavani.com -urldelivery.com -velen.io -vgoloveboli.net -vielporno.net -vietimgy.pw -vilingstore.net -vkontaktemusic.ru -vsesubwaysurfers.com -vykup-avto-krasnodar.ru -wallpapers-best.com -wapsite.me -watchinf.com -wdrake.com -wesharepics.com -wildcattube.com -wsgames.ru -x-porno.video -xgames-04.com -xn-----6kcaabbafhu7cskl7akvongwpo7hvjj.xn--p1ai -xn----8sbhefaln6acifdaon5c6f4axh.xn--p1ai -xn--1-8sbcpb0bdm8k6a.xn--p1ai -xn--90acenikpebbdd4f6d.xn--p1ai -xn--e1afanlbnfckd7c3d.xn--p1ai +tula.mdverey.ru +turizmus.us +turvgori.ru +uni.me +unisexjewelry.org +upproar.com +uptime-delta.net +uptimechecker.com +upupa.net +urengoy.pro +valuado.com +viagra.pp.ua +villacoloniale.com +voditeltrezviy.ru +w7s.ru +warningzscaler.heraeus.com +wechatdownload10.com +weekes.biz.tc +wholesalejerseychinashop.com +xlovecam.com +xn--80aaafbn2bc2ahdfrfkln6l.xn--p1ai +xn--hxazdsfy.blogspot.com xn--l1aengat.xn--p1ai -xnxx-n.com -xtube.com -xwatt.ru +yellowstonevisitortours.com yoshkarola.zrus.org -yougame.biz -youradulthosting.com -yun56.co -zeroredirect12.com -zeroredirect9.com +youjizz.com +youthreaders.com +youtubologia.it +zebramart.ru +zeleznobeton.ru diff --git a/dev-tools/test_units/ratelimittest-beta.txt b/dev-tools/test_units/ratelimittest-beta.txt index f0474a154..3656720f2 100644 --- a/dev-tools/test_units/ratelimittest-beta.txt +++ b/dev-tools/test_units/ratelimittest-beta.txt @@ -581,4 +581,14 @@

Welcome to the NginX Ultimate Bad Bot Blocker

This is merely used for testing !!!

+ + + +Welcome to NginX + + +
+

Welcome to the NginX Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ \ No newline at end of file diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index 92ab0df4c..338f1d6f7 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.31 0; +10.1.0.149 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index 4567ab16e..b998dd39c 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -48650,3 +48650,1255 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2490 +### Updated: Thu Jul 29 11:31:25 UTC 2021 +### Bad Bot Count: 617 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ALittle Client +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: Leap +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: ubermetrics-technologies.com +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ + diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 4567ab16e..b998dd39c 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -48650,3 +48650,1255 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2490 +### Updated: Thu Jul 29 11:31:25 UTC 2021 +### Bad Bot Count: 617 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ALittle Client +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: Leap +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: ubermetrics-technologies.com +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ +