From dcb001e0ad5d0f64fda38b04f77bf6f034d3278d Mon Sep 17 00:00:00 2001 From: mitchellkrogza Date: Tue, 27 Jul 2021 14:13:39 +0000 Subject: [PATCH] V4.2021.07.2486 [ci skip] --- AUTO-CONFIGURATION.md | 2 +- MANUAL-CONFIGURATION.md | 2 +- README.md | 2 +- _google_analytics_ghost_spam/README.md | 2 +- .../etc/nginx/conf.d/globalblacklist.conf | 4 +- conf.d/globalblacklist-testing-version.conf | 4 +- conf.d/globalblacklist.conf | 4 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- dev-tools/buildnumber | 2 +- dev-tools/globalblacklist-testing.template | 4 +- dev-tools/globalblacklist.template | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- dev-tools/test_units/blacklist-ips.conf | 2 +- .../random-bots-for-test-quick.list | 20 +- .../test_units/random-bots-for-test.list | 290 ++-- .../random-bots-for-whitelist-test.list | 294 ++-- .../random-referrers-for-test-quick.list | 20 +- .../test_units/random-referrers-for-test.list | 944 ++++++------- .../random-referrers-for-whitelist-test.list | 484 +++---- dev-tools/test_units/whitelist-ips.conf | 2 +- dev-tools/tmp/robots.txt | 1246 +++++++++++++++++ robots.txt/robots.txt | 1246 +++++++++++++++++ 45 files changed, 3570 insertions(+), 1078 deletions(-) diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index dc749f1b6..97de7be88 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2485 +#### Version: V4.2021.07.2486 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [614](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 02c5bcd72..bf74e7290 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2485 +#### Version: V4.2021.07.2486 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [614](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/README.md b/README.md index 67223c960..463412541 100755 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2021.07.2485 +#### Version: V4.2021.07.2486 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [614](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 3f8550340..58ca39f2d 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,7 +3,7 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2021.07.2485 +#### Version: V4.2021.07.2486 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [614](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 28836a0fb..b20314d3d 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2485 -### Updated: Mon Jul 26 14:19:55 UTC 2021 +### Version: V4.2021.07.2486 +### Updated: Tue Jul 27 14:07:44 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 614 ################################################### diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 719d09324..9a0e001dd 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2485 -### Updated: Mon Jul 26 14:19:56 UTC 2021 +### Version: V3.2021.07.2486 +### Updated: Tue Jul 27 14:07:45 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 614 ################################################### diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 28836a0fb..b20314d3d 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2485 -### Updated: Mon Jul 26 14:19:55 UTC 2021 +### Version: V4.2021.07.2486 +### Updated: Tue Jul 27 14:07:44 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 614 ################################################### diff --git a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 719d09324..9a0e001dd 100644 --- a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2485 -### Updated: Mon Jul 26 14:19:56 UTC 2021 +### Version: V3.2021.07.2486 +### Updated: Tue Jul 27 14:07:45 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 614 ################################################### diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 002b55ef2..ca7cdc872 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.193 1; +10.1.0.227 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index fe697a25b..a229fc66b 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.193 0; +10.1.0.227 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 719d09324..9a0e001dd 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2485 -### Updated: Mon Jul 26 14:19:56 UTC 2021 +### Version: V3.2021.07.2486 +### Updated: Tue Jul 27 14:07:45 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 614 ################################################### diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 002b55ef2..ca7cdc872 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.193 1; +10.1.0.227 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index fe697a25b..a229fc66b 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.193 0; +10.1.0.227 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 719d09324..9a0e001dd 100644 --- a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2485 -### Updated: Mon Jul 26 14:19:56 UTC 2021 +### Version: V3.2021.07.2486 +### Updated: Tue Jul 27 14:07:45 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 614 ################################################### diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 002b55ef2..ca7cdc872 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.193 1; +10.1.0.227 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index fe697a25b..a229fc66b 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.193 0; +10.1.0.227 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 719d09324..9a0e001dd 100644 --- a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2485 -### Updated: Mon Jul 26 14:19:56 UTC 2021 +### Version: V3.2021.07.2486 +### Updated: Tue Jul 27 14:07:45 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 614 ################################################### diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index b8c6595ce..a385b0c6e 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -2485 +2486 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index 719d09324..9a0e001dd 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2485 -### Updated: Mon Jul 26 14:19:56 UTC 2021 +### Version: V3.2021.07.2486 +### Updated: Tue Jul 27 14:07:45 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 614 ################################################### diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index 28836a0fb..b20314d3d 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2485 -### Updated: Mon Jul 26 14:19:55 UTC 2021 +### Version: V4.2021.07.2486 +### Updated: Tue Jul 27 14:07:44 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 614 ################################################### diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 9c6b75009..207d50264 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -52.247.250.35 0; +13.89.50.82 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 28836a0fb..b20314d3d 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2485 -### Updated: Mon Jul 26 14:19:55 UTC 2021 +### Version: V4.2021.07.2486 +### Updated: Tue Jul 27 14:07:44 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 614 ################################################### diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 002b55ef2..ca7cdc872 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.193 1; +10.1.0.227 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index fe697a25b..a229fc66b 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.193 0; +10.1.0.227 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 28836a0fb..b20314d3d 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2485 -### Updated: Mon Jul 26 14:19:55 UTC 2021 +### Version: V4.2021.07.2486 +### Updated: Tue Jul 27 14:07:44 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 614 ################################################### diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 002b55ef2..ca7cdc872 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.193 1; +10.1.0.227 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index fe697a25b..a229fc66b 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.193 0; +10.1.0.227 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 28836a0fb..b20314d3d 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2485 -### Updated: Mon Jul 26 14:19:55 UTC 2021 +### Version: V4.2021.07.2486 +### Updated: Tue Jul 27 14:07:44 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 614 ################################################### diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 002b55ef2..ca7cdc872 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.193 1; +10.1.0.227 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index fe697a25b..a229fc66b 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.193 0; +10.1.0.227 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 28836a0fb..b20314d3d 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2485 -### Updated: Mon Jul 26 14:19:55 UTC 2021 +### Version: V4.2021.07.2486 +### Updated: Tue Jul 27 14:07:44 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 614 ################################################### diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 9c6b75009..207d50264 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -52.247.250.35 0; +13.89.50.82 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index bc1f509b4..680f34474 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2485 -### Updated: Mon Jul 26 14:19:55 UTC 2021 +### Version: V4.2021.07.2486 +### Updated: Tue Jul 27 14:07:44 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 614 ################################################### diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 9c6b75009..207d50264 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -52.247.250.35 0; +13.89.50.82 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 638e45671..274f5c58d 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2485 -### Updated: Mon Jul 26 14:19:55 UTC 2021 +### Version: V4.2021.07.2486 +### Updated: Tue Jul 27 14:07:44 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 614 ################################################### diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index 002b55ef2..ca7cdc872 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.193 1; +10.1.0.227 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index cece8fae5..800924532 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -ADmantX -Dotbot -FHscan -Internet Ninja -Moblie Safari -Musobot -Net Vampire -SuperHTTP -WebCopier -evc-batch +Blow +CherryPicker +CopyRightCheck +GetRight +JustView +Mr.4x3 +Nameprotect +Semrush +seobility +sysscan diff --git a/dev-tools/test_units/random-bots-for-test.list b/dev-tools/test_units/random-bots-for-test.list index 26fc711af..2c45f3100 100644 --- a/dev-tools/test_units/random-bots-for-test.list +++ b/dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ 01h4x.com +404checker +404enemy 80legs -Abonti +Aboundex Acunetix -AfD-Verbotsverfahren -AhrefsBot -Alexibot -AllSubmitter -Alligator AlphaBot -Anarchy -Anarchy99 -Anthill +Anarchie Apexoo -AwarioRssBot +Attach AwarioSmartBot +BBBike BDCbot +BackDoorBot +BackStreet +BackWeb BacklinkCrawler -Barkrowler -BlackWidow -Blackboard +BatchFTP +Bitacle +Black Hole +Blow Boardreader +BotALot Brandprotect -Brandwatch BuiltBotTough -BuzzSumo -CCBot CSHttp -Calculon Cegbfeieh CheTeam -CheeseBot -ChinaClaw -Cliqzbot +CherryPicker +Chlooe +Claritybot Cloud mapping Cogentbot -Collector Copier -CopyRightCheck +Craftbot CrunchBot -Custo -DIIbot DTS Agent -DatabaseDriverMysqli -Deusu -Devil +Demon +Dirbuster Disco -DittoSpyder -DomCopBot -DomainAppender +Dispatch +DnyzBot DomainCrawler -Dotbot +DomainSigmaCrawler +DomainStatsBot +Domains Project Dragonfly -Drip -ECCP/1.0 +EMail Siphon +EasyDL Ecxi -EroCrawler -Exabot +Evil ExtLinksBot -ExtractorPro -Ezooms -FemtosearchBot +Extreme Picture Finder +FHscan +Fimap Firefox/7.0 FlashGet -Foobot FrontPage -Fuzz +FyberSpider Fyrebot -G-i-g-a-b-o-t GT::WWW -GetWeb -Gigabot -GoZilla +GalaxyBot +Getintent +Gigablast +Go!Zilla +Go-Ahead-Got-It +GrabNet GrapeFX -GridBot -HTMLparser -Harvest -Havij +Haansoft +Heritrix HybridBot -IDBot -IndeedBot -InfoNaviRobot -InfoTekies +Iblog +Image Fetch InterGET -InternetSeer -Iria +IstellaBot JamesBOT -Jbrofuzz JetCar +Jetty Joomla +Jorgee +Jyxobot Keyword Density -Kinza -Lanshanbot -LeechFTP LeechGet -Lftp LibWeb -Libwhisker +LieBaoFast Lightspeedsystems Likse +LinkScan +Linkbot +Linkdexbot LinkextractorPro LinkpadBot +Lipperhey +Lipperhey Spider +Litemage_walker +Lmspider Ltx71 -MFC_Tear_Sample -MQQBrowser -MSFrontPage -MTRobot Mag-Net Majestic SEO -Majestic-SEO -MarkMonitor -Masscan -Mata Hari -Meanpathbot -MicroMessenger +MarkWatch +MeanPath Bot +MegaIndex.ru +Microsoft Data Access Microsoft URL Control -Mister PiX +Moblie Safari +Mojeek Mojolicious -Musobot +Msrabot Nameprotect -NearSite -Needle -Nessus +Navroad Net Vampire NetAnts -Netvibes -Nuclei -Nutch -Offline Explorer +NetMechanic +NetSpider +Netcraft +NextGenSearchBot +Niki-bot +Nimbostratus +Nmap Offline Navigator OnCrawl +OpenVAS Openfind -Openvas -OrangeBot OrangeSpider OutclicksBot -PageGrabber -Pandalytics -Panscient -Petalbot -Piepmatz -Pixray -ProWebWalker -Probethenet +PHPCrawl +POE-Component-Client-HTTP +PageThing.com +Papa Foto +PeoplePal +Picsearch +PictureFinder +Pimonster +PleaseCrawl +ProPowerBot Pu_iN -Pump PxBroker -QueryN Metasearch -RSSingBot -RankActiveLinkBot +PyCurl +Quick-Crawler +RankActive +RankFlex RankingBot +Rankivabot RankurBot -Re-re -RealDownload RebelMouse -Ripper +RedesScrapy +RepoMonkey RocketCrawler -Rogerbot +SEOkicks SEOkicks-Robot -SEOlyticsCrawler -SEOstats +SMTBot SalesIntelligent -Scrapy +ScoutJet +Screaming ScrepyBot -SemrushBot -SeoSiteCheckup +Searchestate +SearchmetricsBot +Semrush SeobilityBot Seomoz Shodan +Siphon Site Sucker +SiteExplorer SiteLockSpider -SiteSnagger -SiteSucker +Sitebeam Siteimprove +Sitevigil SlySearch -SmartDownload -Snake Snoopy SocialRankIOBot Sogou web spider -Sosospider -SpaceBison -Spanner +Sottopop +Spammen SputnikBot +Sqlmap +Sqlworm Steeler -Sucuri SuperBot +Surfbot +Suzuran Swiftbot -Szukacz T8Abot -Telesoft -Telesphoreo -The Intraformant +Teleport +Telesphorep TightTwatBot -Titan -Trendictionbot -Typhoeus -URLy Warning -URLy.Warning +Toweyabot +Tracemyfile +Twice V-BOT -VoidEYE -WISENutbot +Vacuum +Vagabondo +VeriCiteCrawler +VidibleScraper +Virusdie +WBSearchBot +WEBDAV WPScan +WWW-Collector-E +WWW-Mechanize WWW::Mechanize -WWWOFFLE +Wallpapers/3.0 Web Enhancer -Web Fetch -Web Sucker +Web Pix WebAuto -WebCopier -WebFuck +WebBandit +WebCollage +WebEnhancer WebGo IS WebLeacher WebPix -WebSauger -WebSucker +WebStripper WebWhacker WebZIP +WebsiteExtractor WebsiteQuester +Webster Whack Whacker -WinHTTrack -Zade +Widow +Wonderbot +Woobot +Wotbox +Xenu +Zauba +Zermelo Zeus -ZumBot -arquivo-web-crawler +ZmEu arquivo.pt check1.exe -crawl.sogou.com -crawler.feedback -crawler4j -domainsproject.org -evc-batch +cognitiveseo +com.plumanalytics +gopher heritrix instabid -magpie-crawler +ips-agent +lwp-trivial meanpathbot netEstate NE Crawler -oBot -ripz -satoristudio.net -seoscanners -sexsearcher +page scorer +plumanalytics +scan.lol siteripz sogouspider -sp_auditbot +trendiction.de voyagerx.com +x09Mozilla x22Mozilla -xpymep1.exe zauba.io +zgrab diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 91fae2081..2a8970e1a 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ +360Spider 404checker -404enemy 80legs -ADmantX ASPSeek -Aboundex -AhrefsBot -AllSubmitter -Alligator +Acunetix +AfD-Verbotsverfahren +Aipbot AlphaBot -Apexoo -BackDoorBot -BacklinkCrawler -Battleztar Bazinga -BetaBot -Bigfoot +Ankit +Aspiegel +Asterias +Attach +BBBike +BDCbot +BackStreet +Badass +Barkrowler Bitacle -Blow -BlowFish -Bolt -Brandprotect -BuiltBotTough +Black Hole +BlackWidow +Boardreader +Buddy BuiltWith +BunnySlippers BuzzSumo -Calculon -Cegbfeieh -CensysInspect -CheTeam +CODE87 +CSHttp CheeseBot -Chlooe -Collector +Cloud mapping +Cocolyzebot +CopyRightCheck CrazyWebCrawler CrunchBot Curious -CyotekWebCopy -DBLBot -DIIbot +DataCha0s Demon -Devil Digincore -Dirbuster -Discoverybot -DomainAppender -DomainCrawler +Disco +Dispatch +DnyzBot DomainSigmaCrawler +DomainStatsBot Domains Project -Dotbot -Dragonfly -EMail Wolf +Drip +EMail Siphon +EasyDL Ebingbong -EroCrawler -Exabot +Ecxi +EirGrabber +Evil Express WebPictures ExtLinksBot Extreme Picture Finder -Ezooms FDM -FHscan -Firefox/7.0 -Flunky Freeuploader -Fuzz +FyberSpider +Fyrebot Genieo -GetWeb Getintent -GoZilla GrabNet Grabber -GrapeFX -HMView +Grafula +GrapeshotCrawler HTTP::Lite +Harvest Heritrix Hloader Humanlinks HybridBot +Iblog Id-search -IlseBot Image Fetch Image Sucker IndeedBot +InfoNaviRobot InfoTekies -Intelliseek InterGET -Iria +Internet Ninja IstellaBot -Jbrofuzz -JennyBot -JetCar -Jetty +JamesBOT JikeSpider -Kozmosbot -LNSpiderguy -LibWeb +Joomla +JustView +Kenjin Spider +Kinza +Lanshanbot +Larbin +LeechFTP +LexiBot +Lftp LieBaoFast -Lightspeedsystems -Likse -LinkextractorPro -LinkpadBot -LinksManager +LinkScan +Linkbot LinqiaMetadataDownloaderBot -Lipperhey Spider -Lmspider -MIDown tool -MSFrontPage +LinqiaScrapeBot +Lipperhey +MQQBrowser MSIECrawler -Mag-Net +MTRobot Magnet +Mail.RU_Bot Majestic SEO -Mass Downloader -Masscan -Mata Hari +Majestic12 +MarkMonitor +MarkWatch MauiBot -MeanPath Bot -Mediatoolkitbot +Meanpathbot +Metauri +MicroMessenger Microsoft Data Access +Microsoft URL Control Moblie Safari -Mojeek -Mozlila -NPbot +Mojolicious +Morfeus Fucking Scanner Nameprotect NearSite +Needle Nessus -NetMechanic -NetSpider -Netcraft -Nettrack -NextGenSearchBot -Niki-bot -Nikto -NimbleCrawler +Net Vampire +NetAnts Nimbostratus +Nmap Nuclei -Nutch +Octopus Offline Explorer OnCrawl -OpenLinkProfiler -Openfind OrangeBot -OrangeSpider +OutfoxBot PECL::HTTP -PHPCrawl -PageScorer -PageThing.com +Page Analyzer +PageAnalyzer Pandalytics +Papa Foto Petalbot Pi-Monster Picscout -Picsearch +PictureFinder Piepmatz -Pixray Pockey -ProPowerBot -Probethenet -Psbot -PxBroker -Quick-Crawler -RSSingBot -RankFlex -RankingBot -Re-re -ReGet +ProWebWalker +Pu_iN +PyCurl +RankActiveLinkBot +RankingBot2 RealDownload +Recorder RedesScrapy -RepoMonkey -SBIder -SEOkicks-Robot +Rogerbot +SEOkicks SEOstats -SalesIntelligent +Screaming +SearchmetricsBot +Semrush SentiBot +SeoSiteCheckup +Seomoz +Shodan +Siphon Site Sucker SiteExplorer -Sitebeam -SlySearch +SiteLockSpider +SiteSnagger +SiteSucker +Siteimprove +SmartDownload Sociscraper -Sogou web spider -Spammen -SpankBot -Spinn3r +Sottopop +SputnikBot +Sqworm Steeler -Sucuri +Stripper SuperBot +SuperHTTP +SurveyBot Suzuran -T0PHackTeam +Swiftbot T8Abot -TeleportPro -TightTwatBot +Teleport +Telesoft +Telesphoreo +Telesphorep +The Intraformant Titan Toata -Toweyabot +Tracemyfile +Trendictionbot True_Robot -Turingos -Twice -Upflow -V-BOT +Turnitin +TurnitinBot +UnisterBot +VCI Vacuum -Vagabondo -VidibleScraper -Virusdie -VoidEYE +Voil Voltron WASALive-Bot WBSearchBot +WEBDAV WISENutbot +WPScan Web Auto -Web Collage Web Fetch Web Fuck Web Sucker -WebAuto -WebFetch -WebFuck -WebGo IS -WebLeacher +WebBandit +WebCollage WebReaper +WebSauger +WebStripper +WebSucker +WebWhacker +WebZIP Webalta -WebsiteExtractor -Webster +WebmasterWorldForumBot +WebsiteQuester +Whack Who.is Bot Widow +WinHTTrack WiseGuys Robot -Wonderbot Wotbox +Wprecon +Xaldon WebSpider +YoudaoBot Zade -Zermelo +Zauba Zeus -ZoominfoBot -ZumBot +ZmEu +ZyBorg archive.org_bot -arquivo-web-crawler +arquivo.pt +check1.exe cognitiveseo crawl.sogou.com -facebookscraper -gopher -instabid +domainsproject.org +heritrix internetVista monitor -ips-agent -magpie-crawler +isitwp.com +lwp-request mediawords +muhstik-scan page scorer plumanalytics probe-image-size +scan.lol +seobility seostar -sysscan +sitechecker.pro +sp_auditbot tAkeOut -trendiction.com -voyagerx.com -webmeup-crawler -x09Mozilla xpymep1.exe -zauba.io diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 812ea3aff..1c14ae00d 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -aviva-limoux.com -capacitacionyaprendizaje.com -electricwheelchairsarea.com -fidelityfunding.com -keywords-monitoring-your-success.com -muizre.ru -sex-znakomstva.online -sexs-foto.top -sportbetfair.com -tonerbox.kz +businescoop.men +calstaterealty.us +christianlouboutinsaleoutletonline.us +mediafresh.online +myiptest.com +online-x.ru +photosaga.info +riralmolamsaca.tk +suture.co +vut.com.ru diff --git a/dev-tools/test_units/random-referrers-for-test.list b/dev-tools/test_units/random-referrers-for-test.list index bae97a08f..31678dd84 100644 --- a/dev-tools/test_units/random-referrers-for-test.list +++ b/dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -101lesbian.xyz -20pascals.nl -256bit.by -2728fb936f0.com -404.mn -4replicawatch.net -57883.net -6hopping.com -7zap.com -83net.jp -883zy.com -abovetherivernc.com -adbetclickin.pink -adconscious.com -addbin.men -adidas.frwebs.fr -admeasures.com -adult-shop.com.ua -adultfriendfinder.com -ahhjf.com -akita.kz -aktobe.xkaz.org -albumsuper.info -alfapro.ru -ampower.me -analytics-ads.xyz -anaseracresar.tk -animaltoplist.com -anmysite.com -antons-transporte.de -apccargo.com -apple.com-webbrowsing-security.science -approved.su -apxeo.info -aquarium-pro.ru -artdeko.info -artdestshop.eu -artefakct.com -asscenihotosidea.blogspot.com -atmovs.com -atofilms.com -bankofthewext.com -bbtec.net -beachfix.co -bestadbid.com -bestdraws.com -besthoro.ru -bestssaker.com -bestwebsiteawards.com -bigames.online -bimmerpost.com -bio.trade-jp.net -bjetjt.com -blog.f00kclan.de -blog.koreadaily.com -boanonihaca.tk -bobinoz.com -bonky.biz -brainboosting.club +12bet.com +12masterov.com +3rbseyes.com +3xforum.ro +8lufu.com +abcguru.xyz +acmebtn.ml +activecampaign.dreamhosters.com +addtoadd.men +adman.gr +adserver-e7.com +adultmeetonline.info +affiliate-fr.com +allornamenti.com +alphacarolinas.com +amospalla.es +ananumous.ru +animalrank.com +anticrawler.org +apartamentwroclaw.eu +apiportalmorecom-a.akamaihd.net +arabgirls.adultgalls.com +arewater.com +arquapetrarca.info +artavenuegardenstudios.com +artel116.ru +arvut.org +asseenontv.ru +autochoixspinelli.com +autogrep.ru +autoloans.com +avramstroy.ru +avto-oligarh.ru +azartmsl.com +backuperwebcam.weebly.com +baditri.com +baladur.ru +bang-hotties.com +bayareaaeroclub.org +beachdriveblog.com +best-businessman.ru +bestcurs.org +bestempresas.es +bestmouthwash.club +bestpriceninja.com +bestwaystogetpaid.us +bikini-image.com +bkgr.se +blacktwhite.com +blavia.00author.com +bongacams.com +bonzibuddi.com +bracketsmackdown.com +bradleylive.xyz +brainzpod.win +brandbucket.com brandedleadgeneration.com -bristolhotel.com.ua -budilneg.xyz -buy-forum.ru -buybest1.biz -cactussoft.biz -cannibalcheerleader.com -canvas.gloverid.site +brandehk.dk +bugof.gq +buqayy0.livejournal.com +businesn.men +buttons-for-your-website.com +buynorxx.com +bzero1jewelry.net +cakemediahq.com.au +callejondelpozo.es +calstaterealty.us +cand.jp +cartierlovestore.com +cartierreplica.pw +cartierring.xyz cbcseward.com -cbozhe.com -ceotrk.com -cfsrating.sonicwall.com -chinajerseyswholesalecoupons.com -chiptuninger.com -chlooe.com -cialisndbrx.com -clickaider.com -clipartnew.com -cocolyze.com -com-supportcenter.website -communisave.co.za -companies-catalog.com -compliance-alex.top -compliance-donald.xyz -compliance-ivan.xyz -compliance-john.top -contentlook.co -cookielawblog.wordpress.com -cornomase.win -cottageofgrace.com -cracksplanet.com -craftinsta.ru -creditmoney.com.ua -crest3d.ru -cypernhuset.se -danashop.ru -darkages.info +celejihad.info +centre-indigo.org.ua +cfacarrosserie74.com +cheap-pills-norx.com +cheatcode-lita12.rhcloud.com +chelnytruck.ru +chidporn.com +classicakuhni.ru +cleanersoft.com +clicksotrk.com +clothesforcash.com +club-musics.ru +com-onlinesupport.host +compliance-nelson.top +compliance-willy.top +compucelunlock.net +confib.ifmo.ru +copblock.org +copenergo.ru +corta.co +costablue.xyz +cougfan.info +crazyboost.pro +crazyprotocol.com +creditwell.ru +crystalslot.com +cwetochki.ru +dailystrength.org +date-withme.com +daymusam.com +deadlinkchecker.com +dealitright.click decorazilla.com -dekoration.us -dekorkeramik.ru -dev33.dioniqlabb.se +delayreferat.ru +deluxedumps.com +dentfidemountpreach.tk +derevesendeco.com +dev.citetick.com diffbot.com -digitalassetmanagement.site digitalfaq.com -dikqlyremy.info -dinosaurus.site -dipstar.org -directrev.com -djekxa.ru -dktr.ru -domaineaneblanc.com -domznaniy.ru +divisioncore.com +dnepropetrovsk.mistr-x.org +dneprsvet.com.ua +domain-submit.info +domcran.net +dostavka-v-ukrainu.ru dosug-lux.ru -downvids.net -dscaas.website -easycommerce.cf -easync.io -edelstahlschornstein-123.de -edudocs.net -eduinfosite.com -ekspresihati.info -electro-prom.com -emediate.eu -energy-ua.com -eralph.tk -ero-advertising.com -essayassist.com -etotupo.ru -euroskat.ru -evrotekhservis.ru -exchangeit.gq -expdom.com -extlinks.com -fashionindeed.ml -faz99.com -fdzone.org -femdom.twiclub.in -fickblock18.com -fitnesspiks.com -flashahead.info -flowwwers.com -free-fbook-traffic.com -free-share-buttons.top -fsalas.com -fuckmill.com -fungirlsgames.net -futbolkisales.ru -fxtips.ru -g.starmoe.xyz -gaming-journal.com -gaz-voshod.ru -getdot.ru -ggiaro.com -globalscam.ga -gok-kasten.net -golden-praga.ru +download-walpaperhd.blogspot.com +downloadmefiranaratb1972.xpg.com.br +drugspowerstore.com +dstroy.su +duawitchrarato.tk +dumuelave.xyz +e90post.com +ebonyporn.site +ecommerce-seo.com +egovaleo.it +el-nation.com +elektrozigaretten2.yn.lt +eluxer.net +emailaccountlogin.co +emergencyneeds.org +empathica.com +enskedesquashclub.se +erotag.com +erotik-kostenlos.net +erotikchat-24.com +evaashop.ru +exchanges-bet.com +expertblog.info +exportshop.us +extrabot.com +extremepornos.net +extremez.net +eywords-monitoring-your-success.com +ezigarettenshop2.myblog.de +f1nder.org +fatfasts-4tmz.com +fatmaelgarny.com +favoritemoney.ru +femmesdenudees.com +ff30236ddef1465f88547e760973d70a.com +fickenprivat.info +filmbokep69.com +financetip.eu +fishingwholesale.us +forensicpsychiatry.ru +formulaantiuban.com +forums.d2jsp.org +fotoxxxru.com +foxydeal.com +frameimage.org +funcrushgames.com +galaxys6manual.info +gameplexcity.com +gamezblox.com +gate5.co.za +gay-site.store +gembird.com +geoads.com +germes-trans.com +get-free-traffic-now.com +getaclueamerica.com +getoutofdebtfree.org +given2.com +glcomputers.ru +global-ics.co.za +glogow.pl +golfresa.lucania.se +gombita.info goodwinmetals.co -googglet.com -goosefishpost.bid -greenshop.su -gribkovye-zabolevaniya.com -gtopstats.com -hccoder.info -hdwalls.xyz -healthcarestore.info -heartofpayne.xyz -heliko.no -herehloadibs.cf -hermesreplica.pw -hidefiles.org -hkladys.com -hoholikik.club -hollywoodactress.info -homelygarden.com -homemade.gq -homemature.net -homik.pw -hopeonthestreet.co.uk -houston-vikings.com -hvd-store.com -iamsport.org -icaseclub.ru -ifirestarter.ru -igadgetsworld.com -impresagaia.it -incep.imagine-ex.co -inet-traffic.com -infazavr.ru -infektsii.com -infostatsvc.com +goodwriterssales.com +gosreg.amchs.ru +gox.com.ua +graphicwe.org +growmyfunds.ca +gsasearchenginerankerhelp.com +gsmlab.pl +handicapvantoday.com +hanink.biz.ly +hdapp1008-a.akamaihd.net +hdwallpapers-free.com +heatpower.ru +highspeed5.net +histats.com +hotblog.top +hoterika.com +hotgirlhdwallpaper.com +hothor.se +https-legalrc.biz +hut1.ru +i-midias.net.br +id-forex.com +ideoworld.org +impisr.edunsk.ru +inboundlinks.win +infobanks.ru +insta-add.pro instakink.com -instasexyblog.com -intermesh.net -internet-apteka.ru -iqoption.pro -isitpaleo.info -janettabridal.com -jbl-charge.info -jerseywholesaleelitestore.com -jewelryandfiligree.com -jwcialislrt.com -karting196.ru -kerch.site -ketoanhanoi.info -keywordteam.net -khadastoafarde.tk -kinky-fetishes.com -kino2018.cc -kinocccp.net -kinomaniatv.pl -kiwe-analytics.com -koleso24.com.ua -komputernaya-pomosh-moscow.ru -kovesszucs.atw.hu -krasivoe-hd.net -lalalove.ru -lechenie-gemorroya.com -leftofcentrist.com -legalrc.biz -lenpipet.ru -lida-ru.com -life.biz.ua -link.ac -liquimondo.com -littlesexdolls.com -littlesunraiser.com -lkbennettoutlet.online -loftdigital.eu -long-beach-air-conditioning.com -lookingglassemb.com -lostaruban.com -luciddiagnostics.in -luxup.ru -m3gadownload.pl -m5home.ru -macdamaged.space -mailemedicinals.com -mamasuper.prom.ua -manve.info -marktforschung-stuttgart.com -marmotstore.online -matematikus.info -medicine-4u.org -members.ghanaweb.com +intim-uslugi.info +invest-pamm.ru +iptool.xyz +izhevsk.xrus.org +jackpotchances.com +javatex.co.id +javitas.info +javstock.com +jerseychinabizwholesale.com +jillepille.com +johannesburgsingles.co.za +kaac.ru +kakadu-interior.com.ua +kaliningrad.zrus.org +karusel-market.ru +kazinogames.lv +keyhantercume.com +keywesthideaways.co +keywordhouse.com +kicknights.gq +kidskunst.info +kladrus.ru +klumba55.ru +kol-energo.ru +koptims.tiu.ru +kostenlos-sexvideos.com +krynica.info +ktotut.net +labplus.ru +lapolis.it +letmacworkfaster.site +lflash.ru +lider-zhaluzi.kiev.ua +lockerz.com +longlifelomilomi.info +louboutinreplica.pw +louisvuittonoutletstore.net +lsex.xyz +lutherstable.org +magicalfind-a.akamaihd.net +mapquestz.us +marcogrup.com +marcowebonyodziezowe.pl +marwer.info +maximpartnerspr.com +medizinreisen.de +medmajor.ru +megavolt.net.ua memberty.com -menhealed.net -meteocast.net -michaelkorssaleoutletonline.net -microsoftportal.net -mikrobiologies.ru +mesbuta.info +mexicosleevegastrectomy.com mindbox.co.za -mmofreegames.online -mmog-play.ru -modabutik.ru -monclerjacketsoutlet.win -monetizationking.net -moneyviking-a.akamaihd.net -moscow.xrus.org -mossmesi.com +mistr-x.org +mobplayer.ru +mobstarr.com +mouselink.co +moviemail-online.co.uk moyaterapiya.ru -musezone.su -nac-bearings.ru -nakrutka.cc -narkologiya-peterburg.ru -nationalbreakdown.com -netvouz.com -nfljerseysforsalewholesaler.com -ngps1.ru -nikkiewart.ru -nina.az -nmrk.ru -normalegal.ru +mozello.ru +mriyadh.com +mustwineblog.com +myftpupload.com +mymobilemoneypages.com +myprintscreen.com +mzdish.site +na15.ru +nagdak.ru +naturalbreakthroughsresearch.com +news-speaker.com +newstaffadsshop.club +newstraveller.ru +newtechspb.ru +newyorkhotelsmotels.info +nl.netlog.com +nodup.ru notasprensa.info -november-lax.com -nyfinance.ml -obiavo.su -offer.camp -oivcvx.website -okel.co -okonich.com.ua -oktube.ru -onclkads.com -one-gear.com +noumeda.com +novosibirsk.xrus.org +nowtorrents.com +o-o-8-o-o.ru +obiavo.ru +obnal.org +ochistka-stokov.ru +oecnhs.info +offf.info +oneminutesite.it +onescreen.cc oneshotdate.com +onko-24.com +online-podarki.com online.ktc45.ru -onlinefilmz.net -onload.pw -onthemarch.co -openstat.com -optitrade24.com -oren-cats.ru -orsonet.ru -outpersonals.com -painting-planet.com -paleohub.info -panchro.xyz -parajumpersoutlet.online -partner-stop.men -paydayonlinecom.com -pc-services.ru -pdamods.ru -penisvergrotendepillennl.ovh -pensplan.com -perso.wanadoo.es -petedrummond.com -picturesmania.com -pipki.r.acdnpro.com +onstrapon.purplesphere.in +osagonline.ru +paceform.com +pateaswing.com +pdns.download +pendelprognos.se +perfection-pleasure.ru +pharmacyincity.com +phormchina.com +photochki.com +photokitchendesign.com +phrcialiled.com +picsfair.com +pinup-casino1.ru +piuminiita.com pixshark.com -pogruztehnik.ru -popander.mobi -popunder.net -pornhive.org -pornhubforum.tk -pornoslive.net +pizza-tycoon.com +pkr1hand.com +play-mp3.com +play.leadzupc.com +playtap.us +pod-muzyku.club +polybuild.ru +popmarker.com +poppen-nw.net +poptool.net +pornoblood.com +pornofeuer.com +pornohd1080.online +pornosexrolik.com +pornotubexxx.name +pornozhara.com portadd.men -portside.xyz -poshiv-chehol.ru -predominant-invent.tk -pregnant.guru -preparevideosafesystem4unow.space -priceg.com +pricheskaonline.ru printie.com -privatbank46.ru +prod2016.com prodess.ru -profitfx.online -profolan.pl -projectforte.ru -pshare.biz +projefrio.com.br +prostitutki-astana.org +prostitutki-yaroslavlya.org psoriasis-file.trade -ptr.ruvds.com -pulseonclick.com -purplestats.com -putitin.me -puzzleweb.ru -questionmarque.ch -quick-offer.com -quickcashlimited.com -quit-smoking.ga -qwesa.ru -rabot.host.sk -rabotaetvse.ru -rangoman.date -ranksignals.com -ranksonic.com -real-time-analytics.com -realizmobi.com -realtytimes.com -recinziireale.com -reimageplus.com +qexyfu.bugs3.com +quangcaons.com +rainbowice.ru +rank-checker.online +ranksonic.info +raschtextil.com.ua +rcpmda.ikan1080.xyz +reeyanaturopathy.com +remmling.de remont-ustanovka-tehniki.ru -renecaovilla.online -renewablewealth.com -replica-watch.ru -rezeptiblud.ru -riders.ro -roll123.com -ruex.org.ua -rutor.group -s-iwantyou.com -sack.net -sardinie.us -satoristudio.net -saveriopiazza.it -sayyoethe.blogspot.co.za -sbtdesign.co.uk -scanner-alex.top -scanner-barak.top -scanner-george.top -scanner-jack.top -scanner-jessica.top -scanner-josh.top -scanner-margo.top -scanner-mark.top -school-diplomat.ru -screentoolkit.com -sdi-pme.com -searchinquire.com -seorankinglinks.us -service-core.ru -sevenstars7.com -sex-dating.co -sex-znakomstva.online -sexblog.pw -sexkontakteao.info -sexkrasivo.net -sexpornotales.net -sexsearch.com -sexspornotub.com -shoppingmiracles.co.uk -shopvilleroyboch.com.ua -shtaketniki.kz -siteimprove.com -slavic-magic.ru -slkrm.ru +rique.host.sk +ruscoininvest.company +rusenvironmental.net +ruspdd.com +russia-tao.ru +rustic-quiver.win +ryetaw.com +sady-urala.ru +saitevpatorii.com +sarf3omlat.com +scanner-donald.top +scanner-nelson.top +scanspyware.net +scottbywater.com +searchimage.co +seo-prof1.xyz +seofied.com +seoprofiler.com +sevendays.com.ua +sex-pr.net +sex-sex-sex5.com +sexad.net +sexyali.com +sexystrippe.info +sexyteens.hol.es +sexytrend.ru +shemalegalls.blogporn.in +sibvitr.ru +signal03.ru +silvermature.net +simple-share-buttons.com +sindragosa.comxa.com +skatestick.bid +sklad-24.ru +skutecznetabletkinaporostwlosow.pl +slowmac.tech +slujbauborki.ru +smart-balancewheel.com smartbalanceworld.com -snow.nvr163.com -soft1.ru -soietvousmaime.fr -sosdepotdebilan.com -spacash.com -space-worry.ml -spammen.de -ssconstruction.co +smartpet.ru +snegozaderzhatel.ru +social-button.xyz +socialseet.ru +socialtrade.biz +sockshare.net +softomix.com +sophang8.com +sortthemesitesby.com +sotechco.co +sotkal.lark.ru +spitfiremusic.com startwp.org -statustroll.com -store-rx.com -streetfire.net -stromerrealty.com -success-seo.com -supers.com.ua -surflinksmedical.com -swinger-mobil.net -swinon.site -tackletarts.co -taqywu51.soup.io -tattooreligion.ru -taxi-v-eisk.ru -td-l-market.ru -templates.franklinfire.co -thaismartloan.com -thegalerie.eu -thesmartsearch.net +starwars.wikia.com +steamoff.net +stockspmb.info +strana-krasoty.ru +sucsesofinspiration.com +sukarame.net +superiends.org +surintech.ac.th +svarkagid.com +szamponrevita.pl +taaaak.com +tabakur77.com +tehngr.ru +thecounter.com +thexart.club thfox.com -thiegs.reco.ws -timetorelax.biz -tizanidine4mgprice.blogspot.com -tizanidineandgabapentin.blogspot.com -tizanidinenarcotic.blogspot.com +timeallnews.ru +tivolibasket.it +tizanidineandcipro.blogspot.com +tizanidineandhydrocodone.blogspot.com +tizanidinecapsules.blogspot.com +tizanidinedruginteractions.blogspot.com +tizanidinehcl2mg.blogspot.com +tizanidinerecreationaluse.blogspot.com +token-lab.org toolsky.com -toon-families.com -tooplay.com -topshef.ru -torrents.cd -totu.us -tradedeals.biz -traderzplanet.in -traffic-club.info -trafficreceiver.club -truck-land.life -tversvet.ru -tyumen.xrus.org -uac.net.au -ugogo.info -unacittaconte.org -uni.me +tootoo.to +top-l2.com +topflownews.com +topvidos.ru +trackzapper.com +trafficshaper.com +traffixer.com +trkur.com +trustl.life +turist-strani.ru +turn-up-life.life +twinderbella.com +twittrading.com +uamtrk.com +ucoz.ru +ugguk.online unrealcommander.org -us-america.ru -usadacha.net -uytmaster.ru +url-extractor.xyz +v-doc.co valuado.com -veopornogratis.xxx -viagra.pp.ua -viagraneggrx.com +vduplo.ru +veselokloun.ru +vezuviy.su +video--production.com videogamesecrets.com -vinsit.ru -vipps.com.my -vipsexfinders.com -vodkoved.ru -vrnelectro.ru -vse-pesni.com -vsesubwaysurfers.com -vzlomfb.com -vzlomtw.com -w7s.ru -wallpapersdesk.info -waterefficiency.co -webalan.ru -webkeyit.com -webnode.me -webtherapy.ru -wechatdownload10.com -weclipart.com -wedding-salon.net -wholesalecheapjerseysfree.com -wholesalejerseys-cheapest.com -whos.amung.us -worldis.me -worldtraveler.world -wovis.site -www.kabbalah-red-bracelets.com -x-lime.com -xblognetwork.com -xn----8sbdbjgb1ap7a9c4czbh.xn--p1acf -xn--80aagddcgkbcqbad7amllnejg6dya.xn--p1ai -xn--80ab4aa2g.xn--p1ai -xolodremont.ru -xtube.com -xxlargepop.com -xz618.com -yadro.ru -yellowfootprints.com -yougetsignal.com +videokrik.net +vielporno.net +vietimgy.pw +villakohlanta.nu +vip-file.com +vip2ch.com +vladimir.xrus.org +volgograd.xrus.org +voloomoney.com +voloslove.ru +vut.com.ru +vzlomsn.org +watracker.net +wdrake.com +web-analytics.date +webshoppermac.com +website-stealer.nufaq.com +welcomeauto.ru +westen-z.life +whatsappdownload10.com +whengirlsgowild.com +where-toget.com +while.cheapwebsitehoster.com +win-spy.com +works.if.ua +worldmusicfests.com +wormix-cheats.ru +www.solartek.ru +wwwadultcheck.com +wygraj-teraz.com +xn--80aaagvmjabrs1aoc9luc.xn--p1ai +xn--lifehacer-1rb.com +yebocasino.co.za +yourads.website ypmuseum.ru -yubikk.info -yugo-star.ru -zarabotok--doma.ru -zeroredirect1.com -zhuravlev.info -zigarettenonlinekaufen.tumblr.com -zoogdinsney.com -zx6.ru +yuarra.pluto.ro +yunque.pluto.ro +zarenica.net +zeroredirect12.com +zigarettenonlinekaufen2.drupalgardens.com +zlatnajesen.com +zooggames.com +zs2vm.top +zytpirwai.net diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index 48cf01ec9..62421e784 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -01apple.com -5u.com -70casino.online -aac52006ec82a24e08b665f4db2b5013f7.com -abcdeg.xyz -alfa9.com -algarveglobal.com -all4invest.ru -allstatesugarbowl.org -ameblo.top -anabolics.shop -angigreene.com -anticrawler.org -apiluckyleapnet-a.akamaihd.net -apiportalmorecom-a.akamaihd.net -arenda-yeisk.ru -avek.ru -avon-severozapad.ru -avramstroy.ru -azartmix.com -azbuka-mo.ru -baksman.com -bayareaaeroclub.org -bestpriceninja.com -betterscooter.com -bloggedporn.com -brg8.com -businesxxl.com -buy-forum.ru -c-english.ru +1stat.ru +2015god.org +2728fb936f0.com +4istoshop.com +7a2rnuey1tw9ar.ru +abctoppictures.net +adanih.com +adrunnr.com +agahinameh.com +alekseevec.ru +allcredits.su +amyfoxfitness.com +analnoeporno.tv +apibetweenlinesn-a.akamaihd.net +apiwoodensealcom-a.akamaihd.net +appfastplay.com +arius.tech +arquivo.pt +autoplate.info +autoseo-traffic.com +autoua.top +aviav.co +avon-ukraine.com +azbukafree.com +bestfortraders.com +bestkfiledxd.cf +beyan.host.sk +bobba.dzaba.com +bradleylive.xyz +brandehk.dk +briomotor.co +btnativenav.com +buyfriend.ru +c1.onedmp.com +cablecar.us cam-kontakte.org -capture-room.com -carsdined.org -cashkitten-a.akamaihd.net -cdn.walkme.com -changde.58.com -chelyabinsk.xrus.org -cherubinimobili.it -chocolateslim-original.com -clickzzs.nl -cmhomestayagency.com -compliance-barak.top -compliance-mary.top -connectionstrenth.com -cpajunkies.com -data.vtc.pw -deanmoore.ie -demon-tweeks.com -diegolopezcastan.com -dikqlyremy.info -docs4all.com -dogbreedspicture.net -domain-tracker.com -domainsatcost.com -dotnetdotcom.org -dtm-spain.com -dustyorate.com -dverimegapolis.ru -egovaleo.it -elektrozigarettekaufen2.blogse.nl -event-tracking.com -ezigarettenshop1.yolasite.com -fashion-mk.net -fenoyl.batcave.net -fickblock18.com -fickkontakte.org -filesclub.net -filkhbr.com -flashahead.info -fordsonmajbor.cf -forms-mtm.ru -foto-weinberger.at -free-share-buttons.blogspot.com -free-share-buttons.top -freelinkbuilding.website.tk -freza-sverlo.ru -front.to -fxtips.ru -g7m.pl -gambarkatabaru.com -garciniaxt.us -gemara.com -getmiro.com -ghazel.ru -gigablast.com -glass-msk.ru -gloverid.site -gorabagrata.ru -goroda-vsego-mira.ru -hamacapty.com -hermesreplica.win -herokuapp.com -hikesearch.net -hitmuzik.ru +camsex.xxx-cam.webcam +carina-sy.de +cbozhe.com +chatroulette.video +classicakuhni.ru +colobit.biz +compliance-checker-7.info +compliance-walter.top +computernetworksonline.com +converse.ddsoldes.fr +copro.pw +cornomase.win +coslab.club +cpabegins.ru +creditnation.ru +customcatchcan.com +cyprusbuyproperties.com +daooda.com +dfwu1019.info +diarioaconcagua.com +diffbot.com +dnepr-avtospar.com.ua +dood.live +dosug-lux.ru +doublepimp.com +doxysexy.com +earl-brown.info +el-nation.com +elkacentr.ru +enpolis.ru +exmasters.com +exonline.info +exportshop.us +familyholiday.ml +fefo.gdn +fetlifeblog.com +ff30236ddef1465f88547e760973d70a.com +fidelityfunding.com +finansov.info +finemanteam.com +fireads.men +fodelsedagspresenter.nu +franch.info +fxgallery.com +gamewrath.com +gamingspark.com +ge0ip.com +genericviagrasildenafiled.net +geoads.com +get-free-social-traffic.com +gk170.ru +golfresa.lucania.se +gopixdatabase.com +gpms.org.my +gribkovye-zabolevaniya.com +gsasearchenginerankerhelp.com +guiadeserraazul.com +hard-porn.mobi +heartofpayne.xyz +hermesbracelets.xyz hmywwogw.bloger.index.hr -home.myplaycity.com -house.sieraddns.com -houseofrose.com -i4track.net -ilovevitaly.ru -imadedinner.net -imgata.com -iwanttodeliver.com -j-times.ru -jewelryandfiligree.com -joy-penguin.com -junglenet-a.akamaihd.net -kareliatobacco.ru -kinocccp.net +homedecorpicture.us +hotblognetwork.com +hotenergy.ru +imitex-plus.ru +internet-apteka.ru +jobgirl24.ru +jolic2.com +karting196.ru +ketrzyn.pl +kino-filmi.com kmd-pto.ru -ku6.com -kumuk.info -kupit-adenu.ru -kwzf.net -landreferat.ru -laptop-4-less.com +laexotic.com +lalalove.ru +lasvegaslockandsafe.com law-check-eight.xyz -linkarena.com -lion.playtap.us -liveinternet.ro -loadopia.com -m5home.ru -magazintiande.ru -matchpal-a.akamaihd.net -maximpartnerspr.com -maxxtor.eu -medtherapy.ru -members.ghanaweb.com -mesbuta.info -michellblog.online -midst.eu -mojaocena.com -moje-recenze.cz -motherboard.vice.com -moyaterapiya.ru -muizre.ru -my-cash-bot.co -naperehresti.info +ledis.top +levaquin750mg.blogspot.com +liupis.com +localmatchbook.com +lomb.co +lowephotos.info +m292.info +manualterap.roleforum.ru +mararoom.ru +masthopehomes.com +medicineseasybuy.com +mistr-x.org +mmstat.com +moetomnenie.com +moroccosurfadventures.com +mostantikor.ru +mp3downloadhq.com +mrpornogratis.xxx +muz-tracker.net +my-bc.ru +mydownloadengine.com +nagdak.ru narutonaruto.ru -negral.pluto.ro -news-readers.ru -nhl09.ru -nowtorrents.com -obesidadealgarve.com -onlinetvseries.me -painting-planet.com -panchro.xyz -pandarastore.top -partybunny.ru -pastaleads.com +nash-krym.info +nfljerseyscheapbiz.us +nuit-artisanale.com +nyfinance.ml +obiavo.net +olympescort.com +onemantrip.com +oneshotdate.com +onstrapon.purplesphere.in +ooo-gotovie.ru +outpersonals.com pavlodar.xkaz.org -pc-test.net -pepperstyle.ru -perfection-pleasure.ru -pewit.pw -phelissota.xyz -photostudiolightings.com -phpdevops.com -picquery.com -pics-group.com -piluli.info -piulatte.cz -poisk-zakona.ru -popads.net -prchecker.info -privat-girl.net -privatov-zapisi.ru -prlog.ru -profit-opportunity.com +penisvergrotendepillennl.ovh +pensplan.com +perso.wanadoo.es +pflexads.com +phobia.us +photo.houseofgaga.ru +photokitchendesign.com +picsfair.com +pila.pl +pinsdaddy.com +plastjulgranar.se +poiskzakona.ru +porno-home365.com +pornofiljmi.com +pozdravleniya-c.ru +pozdrawleniya.ru +praisong.net +prizesbook.online promoheads.com -pronorm.fr -rada.ru -redirectingat.com -redirectme.net -reelheroes.net -remmling.de -remote-dba.de -rosbalt.com.ua -rostov.xrus.org -rusoft-zone.ru -ryetaw.com -sabaapress.com -sadaholding.com -sarafangel.ru -sarm.tk -search.1and1.com -secretscook.ru -serdcenebolit.com -sextracker.com -sextracker.de -sharebutton.to -silvermature.net -social-fun.ru -space4updating.win -stpicks.com -studiofmp.com -suggest-keywords.com -surgut.zrus.org -surintech.ac.th -svbur.ru -the-usa-games.blogspot.com -themeforest.net -top-karkas.ru -torrents.cd -traffic2cash.org -trafficgenius.xyz -truck-rece.life -tvnewsclips.info -uac.net.au -ufa.xrus.org -ultimateclassicrock.com -unisexjewelry.org -vashsvet.com -veopornogratis.xxx -videochat.life -virus-respirators.com -viteonlusarezzo.it -vmnmvzsmn.over-blog.com -vodkoved.ru -voloslove.ru -vut.com.ru -watch-movies.ru -wcb.su -wesharepics.com -westsextube.com -whatsappbot.flyland.ru -wildworld.site -williamrobsonproperty.com -wovis.site -x5market.ru +propranolol40mg.blogspot.com +pufip.com +purplesphere.in +quickcashlimited.com +rank-checker.online +ranksonic.info +rapidsites.pro +razorweb-a.akamaihd.net +remybutler.fr +rentaremotecomputer.com +replicaclub.ru +rn-to-bsn.com +rocis.site +rocketchange.ru +rockingclicks.com +rossanasaavedra.net +royalvegascasino.com +rubbed.us +rus-teh.narod.ru +rusexy.xyz +scanner-elena.top +scanner-nelson.top +scmor.ilxc.cc +see-your-website-here.com +senger.atspace.co.uk +seoexperimenty.ru +shivafurnishings.com +shtaketniki.ru +sildenafil-tadalafil.info +sildenafilcitratemed.com +sladkoevideo.com +smartshoppymac.com +space2019.top +spb.ru +stair.registrydomainservices.com +startwp.org +sterva.cc +stjamesschool.info +sugarlyflex.pw +suhanpacktech.com +swinon.site +t3chtonic.com +tako3.com +tamada69.com +testingads.pro +theallgirlarcade.com +thepantonpractice.co.uk +tiens2010.ru +tivolibasket.it +tizanidinenarcotic.blogspot.com +tizanidineoral.blogspot.com +tizanidinerestlesslegsyndrome.blogspot.com +top1-seo-service.com +topbestgames.com +topsy.com +track2.shop +tracklead.net +tracxn.com +traffique.net +tvand.ru +tvorozhnaja-zapekanka-recept.ru +twittrading.com +u555u.info +unrealcommander.org +vchulkax.com +video-hollywood.ru +videochat.mx +videogamesecrets.com +vigrx-original.ru +virtuagirl.com +vseigry.fun +vucms.com +weightatraining.com +welcomeauto.ru +whereverdesperate.gq +wholesalejerseyscheapjerseys.us.com +xboxster.ru xkaz.org -xn----7sbaaabaei0cc8aj5bj0bncejx.xn--p1ai -xn----7sbho2agebbhlivy.xn--p1ai -xn--80aaagvmjabrs1aoc9luc.xn--p1ai -xn--80aeahghtf8ac5i.xn--p1ai -xn--c1acygb.xn--p1ai -xn--lifehacer-1rb.com -xsion.net -xtube.com -xvideosbay.com -xxart.ru -yachts-cruise.info -yellocloud.be -yousense.info -zaim-pod-zalog-krasnodar.ru -zaloro.com -zambini.ru -zaobao.com.sg -zeroredirect8.com -zigarettenonlinekaufen2.bloog.pl +xn----9sbebi2bvzr7h.xn--p1ai +xn--80aaafbn2bc2ahdfrfkln6l.xn--p1ai +xn--80aaaks3bbhabgbigamdr2h.xn--p1ai +yaoguangdj.com +zaimite.ru +zeroredirect7.com diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index fe697a25b..a229fc66b 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.193 0; +10.1.0.227 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index 5ff118198..717f7d2c5 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -43658,3 +43658,1249 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2486 +### Updated: Tue Jul 27 14:07:45 UTC 2021 +### Bad Bot Count: 614 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ + diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 5ff118198..717f7d2c5 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -43658,3 +43658,1249 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2486 +### Updated: Tue Jul 27 14:07:45 UTC 2021 +### Bad Bot Count: 614 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ +