From 8222098948dcdbe8cce73f8ebc6bb54f54cb7c09 Mon Sep 17 00:00:00 2001 From: mitchellkrogza Date: Mon, 12 Jul 2021 14:11:04 +0000 Subject: [PATCH] V4.2021.07.2468 [ci skip] --- AUTO-CONFIGURATION.md | 2 +- MANUAL-CONFIGURATION.md | 2 +- README.md | 2 +- _google_analytics_ghost_spam/README.md | 2 +- .../etc/nginx/conf.d/globalblacklist.conf | 4 +- conf.d/globalblacklist-testing-version.conf | 4 +- conf.d/globalblacklist.conf | 4 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- dev-tools/buildnumber | 2 +- dev-tools/globalblacklist-testing.template | 4 +- dev-tools/globalblacklist.template | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- dev-tools/test_units/blacklist-ips.conf | 2 +- .../random-bots-for-test-quick.list | 20 +- .../test_units/random-bots-for-test.list | 294 ++-- .../random-bots-for-whitelist-test.list | 280 ++-- .../random-referrers-for-test-quick.list | 20 +- .../test_units/random-referrers-for-test.list | 908 ++++++------ .../random-referrers-for-whitelist-test.list | 484 +++---- dev-tools/test_units/whitelist-ips.conf | 2 +- dev-tools/tmp/robots.txt | 1248 +++++++++++++++++ robots.txt/robots.txt | 1248 +++++++++++++++++ 45 files changed, 3551 insertions(+), 1055 deletions(-) diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 0b867b2f4..3243c64db 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2467 +#### Version: V4.2021.07.2468 #### Bad Referrers Blocked: [7067](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index b3f4bb35c..cb6a9d891 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2467 +#### Version: V4.2021.07.2468 #### Bad Referrers Blocked: [7067](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/README.md b/README.md index f47427fad..e59bb90d7 100755 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2021.07.2467 +#### Version: V4.2021.07.2468 #### Bad Referrers Blocked: [7067](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 801002569..161955f12 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,7 +3,7 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2021.07.2467 +#### Version: V4.2021.07.2468 #### Bad Referrers Blocked: [7067](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index f2019c1a4..6df822f92 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2467 -### Updated: Sun Jul 11 14:05:07 UTC 2021 +### Version: V4.2021.07.2468 +### Updated: Mon Jul 12 14:05:19 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 706738c14..b4d57644d 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2467 -### Updated: Sun Jul 11 14:05:08 UTC 2021 +### Version: V3.2021.07.2468 +### Updated: Mon Jul 12 14:05:19 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index f2019c1a4..6df822f92 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2467 -### Updated: Sun Jul 11 14:05:07 UTC 2021 +### Version: V4.2021.07.2468 +### Updated: Mon Jul 12 14:05:19 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 706738c14..b4d57644d 100644 --- a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2467 -### Updated: Sun Jul 11 14:05:08 UTC 2021 +### Version: V3.2021.07.2468 +### Updated: Mon Jul 12 14:05:19 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index ca0f2829d..0d59e8a61 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.131 1; +10.1.0.90 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 0774f55a5..981829a62 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.131 0; +10.1.0.90 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 706738c14..b4d57644d 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2467 -### Updated: Sun Jul 11 14:05:08 UTC 2021 +### Version: V3.2021.07.2468 +### Updated: Mon Jul 12 14:05:19 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index ca0f2829d..0d59e8a61 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.131 1; +10.1.0.90 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 0774f55a5..981829a62 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.131 0; +10.1.0.90 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 706738c14..b4d57644d 100644 --- a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2467 -### Updated: Sun Jul 11 14:05:08 UTC 2021 +### Version: V3.2021.07.2468 +### Updated: Mon Jul 12 14:05:19 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index ca0f2829d..0d59e8a61 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.131 1; +10.1.0.90 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 0774f55a5..981829a62 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.131 0; +10.1.0.90 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 706738c14..b4d57644d 100644 --- a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2467 -### Updated: Sun Jul 11 14:05:08 UTC 2021 +### Version: V3.2021.07.2468 +### Updated: Mon Jul 12 14:05:19 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index 92a41bad6..1d2ff352b 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -2467 +2468 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index 706738c14..b4d57644d 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2467 -### Updated: Sun Jul 11 14:05:08 UTC 2021 +### Version: V3.2021.07.2468 +### Updated: Mon Jul 12 14:05:19 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index f2019c1a4..6df822f92 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2467 -### Updated: Sun Jul 11 14:05:07 UTC 2021 +### Version: V4.2021.07.2468 +### Updated: Mon Jul 12 14:05:19 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index d55a6b844..c6becb50d 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -20.102.120.177 0; +20.94.225.77 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index f2019c1a4..6df822f92 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2467 -### Updated: Sun Jul 11 14:05:07 UTC 2021 +### Version: V4.2021.07.2468 +### Updated: Mon Jul 12 14:05:19 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index ca0f2829d..0d59e8a61 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.131 1; +10.1.0.90 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 0774f55a5..981829a62 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.131 0; +10.1.0.90 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index f2019c1a4..6df822f92 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2467 -### Updated: Sun Jul 11 14:05:07 UTC 2021 +### Version: V4.2021.07.2468 +### Updated: Mon Jul 12 14:05:19 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index ca0f2829d..0d59e8a61 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.131 1; +10.1.0.90 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 0774f55a5..981829a62 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.131 0; +10.1.0.90 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index f2019c1a4..6df822f92 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2467 -### Updated: Sun Jul 11 14:05:07 UTC 2021 +### Version: V4.2021.07.2468 +### Updated: Mon Jul 12 14:05:19 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index ca0f2829d..0d59e8a61 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.131 1; +10.1.0.90 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 0774f55a5..981829a62 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.131 0; +10.1.0.90 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index f2019c1a4..6df822f92 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2467 -### Updated: Sun Jul 11 14:05:07 UTC 2021 +### Version: V4.2021.07.2468 +### Updated: Mon Jul 12 14:05:19 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index d55a6b844..c6becb50d 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -20.102.120.177 0; +20.94.225.77 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 85d240efb..06a423116 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2467 -### Updated: Sun Jul 11 14:05:07 UTC 2021 +### Version: V4.2021.07.2468 +### Updated: Mon Jul 12 14:05:19 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index d55a6b844..c6becb50d 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -20.102.120.177 0; +20.94.225.77 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 2121275ca..d724d1520 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2467 -### Updated: Sun Jul 11 14:05:07 UTC 2021 +### Version: V4.2021.07.2468 +### Updated: Mon Jul 12 14:05:19 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index ca0f2829d..0d59e8a61 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.131 1; +10.1.0.90 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 35946800d..05e37e599 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -DomCopBot -GrapeFX -IRLbot -PageScorer -Ripper -WEBDAV -Wallpapers/3.0 -Web Fuck -domainsproject.org -sexsearcher +Anarchie +Drip +Ecxi +FrontPage +InterGET +Meanpathbot +Sosospider +WebStripper +ZmEu +crawler.feedback diff --git a/dev-tools/test_units/random-bots-for-test.list b/dev-tools/test_units/random-bots-for-test.list index 38f33ae3f..dd77bd220 100644 --- a/dev-tools/test_units/random-bots-for-test.list +++ b/dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -01h4x.com +404enemy 80legs ADmantX Abonti -Acunetix -AiHitBot +Aboundex +AhrefsBot Aipbot -Alexibot +Anarchie +Anarchy Ankit -Apexoo -Aspiegel Asterias -AwarioRssBot -AwarioSmartBot +Attach +BDCbot +BLEXBot BackWeb BacklinkCrawler -Badass -Bandit +Barkrowler BatchFTP -Bitacle +Blackboard +BlowFish Boardreader +Brandprotect Buck BuiltBotTough BuiltWith Bullseye BunnySlippers -BuzzSumo +CCBot +CSHttp Calculon CazoodleBot -Cegbfeieh CensysInspect -ChinaClaw +CheTeam +CheeseBot +Cliqzbot Collector CopyRightCheck -Cosmos -Crescent -Curious +Copyscape +CrazyWebCrawler Custo CyotekWebCopy -DBLBot -DSearch DTS Agent -DataCha0s -DatabaseDriverMysqli +Demon +DigitalPebble +Dirbuster Disco -Discobot -Discoverybot -DittoSpyder -DomCopBot +DnyzBot DomainAppender -DomainStatsBot -Download Wonder -Express WebPictures +EasyDL +Evil +Exabot ExtLinksBot -Extractor -Extreme Picture Finder -EyeNetIE -Ezooms +FDM +FHscan +FemtosearchBot Fimap -Firefox/7.0 -Foobot -Fyrebot -G-i-g-a-b-o-t +Flunky +Freeuploader +FrontPage +Fuzz GT::WWW -GermCrawler -Gigablast +GalaxyBot +GetRight +GetWeb +Gigabot Go!Zilla -Go-Ahead-Got-It GoZilla +Gotit +GrabNet Grabber GrapeFX -HTTP::Lite -HTTrack Haansoft -HaosouSpider Harvest -Havij +Heritrix Hloader IDBTE4M -Iblog -Id-search +IDBot IlseBot -Image Fetch -JOC Web Spider -JennyBot -Jetty -JustView +IndeedBot +Indy Library +Intelliseek +InterGET +InternetSeer +Iria +IstellaBot +JetCar +Joomla +Jorgee +Jyxobot Kenjin Spider -Kozmosbot -LNSpiderguy +Kinza +Lanshanbot Larbin -LeechGet -LexiBot +LeechFTP Lftp +LibWeb LieBaoFast -Lightspeedsystems -LinkWalker +Likse Linkbot -LinksManager -LinqiaMetadataDownloaderBot -LinqiaRSSBot +LinkpadBot +Lipperhey +Litemage_walker +Lmspider MIDown tool +MIIxpc +MJ12bot MQQBrowser -Magnet +MSFrontPage +MSIECrawler +MTRobot +Mag-Net +Mail.RU_Bot +Majestic SEO +Majestic-SEO Majestic12 -MarkMonitor -Mata Hari -MauiBot -Mb2345Browser Mediatoolkitbot Metauri -Mojeek -Mojolicious +Microsoft Data Access +Microsoft URL Control +Mister PiX Morfeus Fucking Scanner -Musobot -Nameprotect -Navroad +NICErsPRO +NPbot +Name Intelligence NearSite -Needle -NetAnts -NetLyzer -NetMechanic +NetSpider NetZIP -Nettrack -NextGenSearchBot +Netvibes Nibbler -Nmap -Nutch -Octopus +Nuclei Offline Explorer -OnCrawl OpenLinkProfiler Openfind -Openvas -OrangeSpider OutfoxBot PHPCrawl -POE-Component-Client-HTTP -Page Analyzer -PageAnalyzer PageScorer +PageThing.com Panscient -Papa Foto -Petalbot -Pi-Monster -Piepmatz +Picscout +Picsearch +PictureFinder +Pixray PleaseCrawl -ProWebWalker -Pu_iN -QueryN Metasearch +Pockey +ProPowerBot +RankActive +RankActiveLinkBot RankFlex -RankingBot2 -Rankivabot +RankingBot +Re-re ReGet -Reaper -RebelMouse Recorder +RepoMonkey +RocketCrawler +Rogerbot SEOkicks-Robot -SEOlyticsCrawler SEOstats SMTBot ScanAlert +Scrapy +Screaming +ScreenerBot ScrepyBot -Searchestate -SemrushBot -Shodan +Semrush Site Sucker -SiteLockSpider +SiteCheckerBotCrawler +SiteExplorer +SiteSnagger +SiteSucker Sitebeam -Sitevigil -SmartDownload -Snoopy -SocialRankIOBot -Sottopop -SpankBot -Spanner +Siteimprove +SlySearch +Sogou web spider +Spammen Spbot +Spinn3r +SputnikBot +Sqworm Steeler -Stripper -Sucuri SuperBot SuperHTTP Suzuran -Swiftbot T8Abot -TeleportPro Telesoft TheNomad +TightTwatBot Toweyabot -True_Robot +Trendiction +Turingos Turnitin -TurnitinBot +TwengaBot +Twice URLy Warning UnisterBot -Upflow -VCI -Vacuum -Vagabondo -Virusdie +V-BOT +VB Project +VelenPublicWebCrawler VoidEYE -WASALive-Bot +Voil +Voltron +WISENutbot +WWW-Collector-E WallpapersHD WeSEE Web Collage Web Enhancer Web Sauger +Web Sucker +WebAuto WebCollage WebFetch +WebFuck WebGo IS -WebImageCollector -WebLeacher -WebReaper -WebSauger -WebStripper -WebZIP +WebSucker +WebWhacker Webalta -Website Quester -Whacker -Whatweb -Who.is Bot +Whack Widow WinHTTrack WiseGuys Robot -Xaldon_WebSpider -Zade +Wotbox +Wprecon +Zauba ZoomBot +ZumBot +ZyBorg +arquivo-web-crawler +backlink-check crawl.sogou.com -demandbase-bot +crawler.feedback +crawler4j +domainsproject.org eCatch evc-batch -gopher -heritrix -internetVista monitor -isitwp.com -page scorer -pcBrowser -plumanalytics -polaris version -probe-image-size -ripz -satoristudio.net +facebookscraper +lwp-request +magpie-crawler +meanpathbot +mediawords +muhstik-scan +oBot +s1z.ru scalaj-http -scan.lol seoscanners +sexsearcher siteripz -sogouspider +sp_auditbot +tAkeOut trendiction.com -trendiction.de -voyagerx.com xpymep1.exe diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 9c8f9a1ac..40e834305 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -01h4x.com -360Spider -404checker 404enemy +80legs ADmantX -Abonti -AfD-Verbotsverfahren +Aboundexbot +Acunetix AhrefsBot -AiHitBot -Alexibot AllSubmitter -Anarchie +Anarchy +Ankit Apexoo +Aspiegel BDFetch BLEXBot +BackDoorBot BackStreet -Backlink-Ceck -Badass Bandit -BetaBot +Battleztar Bazinga Bigfoot -BlowFish -BotALot -Brandprotect +BlackWidow +Blow +Brandwatch +BuiltBotTough BuiltWith -BunnySlippers BuzzSumo -Cegbfeieh +CazoodleBot +CensysInspect +CheTeam Cloud mapping +Cocolyzebot Cogentbot Collector CopyRightCheck -Copyscape -Curious +Cosmos +CrazyWebCrawler +Crescent +Custo CyotekWebCopy DIIbot -DataCha0s -DatabaseDriverMysqli -Deusu -Dirbuster -Discoverybot +Demon +Devil +Digincore +DigitalPebble +Disco DittoSpyder DnyzBot -DomainSigmaCrawler +DomCopBot +DomainCrawler DomainStatsBot Domains Project -Dotbot -Drip -ECCP/1.0 -EMail Wolf +EasyDL +Ebingbong Ecxi +EirGrabber +EroCrawler +ExtLinksBot ExtractorPro -Extreme Picture Finder Ezooms -FDM -FHscan Fimap -FlashGet Flunky -Freeuploader -FrontPage Fyrebot -G-i-g-a-b-o-t -GT::WWW +GalaxyBot GermCrawler GetRight -Getintent Gigablast -Gigabot +Go-Ahead-Got-It +GoZilla Gotit GrabNet +Grabber Grafula -GrapeFX GrapeshotCrawler +GridBot HMView HTMLparser -HTTP::Lite -HaosouSpider -Heritrix -Hloader -HybridBot -IRLbot +HTTrack +Harvest +Humanlinks +IDBot Iblog -IlseBot +Image Fetch +IndeedBot Indy Library InfoNaviRobot InfoTekies -Internet Ninja -InternetSeer -IstellaBot +Intelliseek +InterGET +Iria JOC Web Spider JamesBOT Jbrofuzz -JennyBot JetCar -Joomla -Kenjin Spider +JustView Lanshanbot -LeechFTP -LeechGet LexiBot -Lftp LibWeb LieBaoFast -Lightspeedsystems -Likse -LinkWalker -Linkbot -Linkdexbot LinkextractorPro -Lipperhey -Lipperhey Spider -Litemage_walker -Lmspider -MSFrontPage -MTRobot -Mail.RU_Bot +MFC_Tear_Sample +MIIxpc +Mag-Net Majestic SEO -Masscan +Majestic-SEO +MarkMonitor +Mass Downloader Mata Hari -Mb2345Browser -Meanpathbot -Mediatoolkitbot +MeanPath Bot MegaIndex.ru MicroMessenger -Microsoft Data Access -Microsoft URL Control -Mister PiX Moblie Safari Mojeek +Morfeus Fucking Scanner Mozlila -Musobot -NetAnts -NetMechanic -NetZIP -Netvibes -NextGenSearchBot -Nimbostratus +Name Intelligence +Nameprotect +Navroad +Nessus +Netcraft +Nettrack Nmap -OpenLinkProfiler -Openfind -OrangeSpider -PECL::HTTP +Nuclei +Offline Explorer +OrangeBot +PHPCrawl POE-Component-Client-HTTP +Page Analyzer PageGrabber -PageScorer -PeoplePal -Picscout +Pi-Monster Picsearch -PictureFinder +Piepmatz Pimonster +Pockey ProWebWalker -Psbot +Probethenet Pump +PxBroker PyCurl +QueryN Metasearch RankFlex -RankingBot +RankingBot2 +RankurBot +Re-re ReGet -Rogerbot +RebelMouse +RedesScrapy +SEOkicks +SEOkicks-Robot SEOlyticsCrawler SEOprofiler SISTRIX Scanbot -Scrapy +ScreenerBot +Searchestate SearchmetricsBot -Semrush -SeoSiteCheckup +SemrushBot +Seomoz +Shodan Siphon +Site Sucker SiteCheckerBotCrawler -SiteLockSpider -SiteSucker +SiteSnagger Sitebeam -Siteimprove -Sitevigil -SlySearch +SmartDownload Snake -Snoopy -SocialRankIOBot +Snapbot Sociscraper SpankBot +Spanner Spbot -Steeler +Sqlworm Sucker -T0PHackTeam -TeleportPro +Sucuri +SuperHTTP +Surfbot +SurveyBot +Szukacz +Teleport Telesphorep The Intraformant TheNomad +TightTwatBot Toata Trendiction Trendictionbot True_Robot Turingos +Turnitin +TurnitinBot +Twice +Typhoeus URLy Warning URLy.Warning -UnisterBot Upflow -V-BOT -VCI -VelenPublicWebCrawler +VB Project +Vacuum +Vagabondo VidibleScraper +Virusdie Voil Voltron -WBSearchBot +WEBDAV WPScan -WWW-Collector-E -WWWOFFLE -Web Enhancer -Web Pix -Web Sauger -Web Sucker +Wallpapers/3.0 +WallpapersHD +Web Collage +Web Fetch +Web Fuck +WebBandit WebCopier WebFetch -WebLeacher +WebFuck +WebGo IS WebPix +WebSauger WebWhacker WebZIP -WebsiteExtractor WebsiteQuester -Webster Whack -Widow -WinHTTrack -Woobot -Wprecon +Who.is Bot +WiseGuys Robot +Wotbox +Xaldon WebSpider +Xenu +Zade Zauba -ZmEu -ZyBorg +Zermelo +Zitebot +ZoominfoBot +archive.org_bot +arquivo-web-crawler backlink-check check1.exe +coccocbot-web +com.plumanalytics +domainsproject.org +eCatch evc-batch -facebookscraper +instabid ips-agent -mediawords +lwp-request netEstate NE Crawler -oBot +pcBrowser +plumanalytics polaris version -ripz -s1z.ru -seoscanners -sitechecker.pro -sogouspider -sp_auditbot +satoristudio.net +scan.lol +seobility +sexsearcher tAkeOut -trendiction.com -webmeup-crawler -x09Mozilla -x22Mozilla -zauba.io -zgrab +trendiction.de +xpymep1.exe diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 259e0e536..c92a42213 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -1qingdao.com -arewater.com -autoloans.com -culad.com -dollartree.info -gate5.co.za -greenshop.su -krasivoe-hd.com -lookingglassemb.com -vedomstvo.net +adtech.us +backlinkwatch.com +conciergegroup.org +costablue.xyz +galleryawesome.com +seoboxes.com +skatestick.bid +stmassage.ru +tizanidinehcl4mgtab.blogspot.com +witherrom55.eklablog.fr diff --git a/dev-tools/test_units/random-referrers-for-test.list b/dev-tools/test_units/random-referrers-for-test.list index b5ddecdfc..6190b0ddd 100644 --- a/dev-tools/test_units/random-referrers-for-test.list +++ b/dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -000free.us -1008.su +00author.com +123locker.com +123movies.love +1bet.com +1flag.co.za 1webmaster.ml -2015god.org -2ads.co.uk +1xbet4you.com +2000k.ru +2020iscoming.info +23kw.ru 4istoshop.com -4pp13.com +51unlim.ru 5u.com -72-news.com -7minuteworkout.com -aa08daf7e13b6345e09e92f771507fa5f4.com -aa14ab57a3339c4064bd9ae6fad7495b5f.com -aa9d046aab36af4ff182f097f840430d51.com -aaa38852e886ac4af1a3cff9b47cab6272.com -abcdefh.xyz -actices.com -acunetix-referrer.com -adbetclickin.pink -adelly.bg -admanaerofoil.com -admeasures.com -adnotbad.com -advancedcleaner.com -adventureparkcostarica.com -aerogo.com -afslankpillen2017nl.eu -agreda.pluto.ro -ahrefs.com -aihelen.net +70casino.online +aa625d84f1587749c1ab011d6f269f7d64.com +above.com +adultnet.in +aero2.ru +agapovdg.ru +agardomains.com +akuhni.by albumsuper.info -alfa9.com -amazingninja.com +alevinefesleri.com +alfabot.xyz +algerianembassy.co.in +all-number.com +allnews.md amazingpic.net -ameblo.jp -ampower.me -anonymizeme.pro -aproposde.com -arabgirls.adultgalls.com -arabsexxxtube.com -arabseyes.com -arenda-yeisk.ru -arpelsreplica.xyz -aruplighting.com +amung.us +an-donut.com +anal-acrobats.com +apartment.ru +apessay.com +appearance-cool.com +arcadepages.com +arenanews.com.ua +arendatora.ru arvut.org -asrv-a.akamoihd.net -astana.xxxkaz.org +ascat.porn +asophoto.com +asrv-a.akamaihd.net atovh.local-finders.com -autonew.biz -aviapanda.ru +autoblogger24.info +autotop.com.ua +autowebmarket.com.ua +aversis.net aviav.co -avirasecureserver.com -avitocars.ru -avon-novosib.ru -b-styles.xyz -babyguns.ru -backiee.com -banksrf.ru -beatifullandscape.co -beauby.ru -becuo.com -bensbargains.net -best-mam.ru -bestdooz.com -bestpriceninja.com -besttorrentknifta.weebly.com -bezcmexa.ru +babyfactory.fr +back.dog +backgroundpictures.net +backlink4u.net +badavit.com.ua +baladur.ru +balkanfarma.ru +baltgem.com +bayareaaeroclub.org +belsetirehafi.tk +bestofferswalkmydogouteveryday.gq +bestplacetobuyeriacta.jw.lt +betterhdporn.com +bezlimitko.xyz +bget.ru big-cash.net -bigames.online -bio-optomarket.ru -bjanshee.ru -bkgr.se -blackhatworth.com -blog.f00kclan.de -bloggen.be -blogrankers.com -bnt-team.com -bochemit.com.ua -boeuklad.com +biketank.ga +bizlist.com.de +bizzliving.com +bjorkbacken.nu +blackle.com +bobinoz.com bolezniorganov.ru -boltalko.xyz -boobsimge.com -bradleylive.xyz -brianhenry.co -brus-vsem.ru -bumble.cheapwebsitehoster.com -bumskontakte.org -buron.pw -business-made-fun.com -buyclomidonlaine.com -buydissertation.net -buyk.host.sk -buypuppies.ca -bwlx.prepedu.cn -byme.se -calcularpagerank.com -calstaterealty.us -capture-room.com -cartierloveringreplica.xyz -cayado.snn.gr -cbb1smartlist12.click -cdnnetwok.xyz -cejewelry.xyz -censys.io -cerev.info -chatseo.com -chcu.net -chorus.terakeet.com -christianlouboutinreplica.pw -cialgenisrx.com -cialischmrx.com -cialisndbrx.com -cicaki.net -cigarpass.com -classiquebijoux.ru -cleansearch.net -clickso.com -cobaltpro.ru -compliance-nelson.top -cookie-law-enforcement-hh.xyz +bouda.kvalitne.cz +bristolhotel.com +bronzeaid-a.akamaihd.net +burger-imperia.com +burger-tycoon.com +buyantiviralwp.com +buyviagraa.blogspot.com +bzero1jewelry.net +c1.onedmp.com +callawaygolfstore.online +carabela.com.do +cardsharp1.ru +cartierbracelet.xyz +cartierlovebraceletreplica.xyz +cashkitten-a.akamaihd.net +cctva.tv +cheapcigarettesc.info +cheapjerseysap.com +cheapwebsitehoster.com +chocolateslim-u-srbiji.com +cialischsrx.com +cialischstgerts.com +clickzzs.nl +clodo.ru +compiko.info +compliance-don.top +compliance-don.xyz +comsysnet.com +connectionstrenth.com coolwallpapers-hd.com -copyrightclaims.org -crackzplanet.com -culad.com -cupday.com -customboxes4less.com -customsua.com.ua -cyprusbuyproperties.com -cyse.tk -daretodonate.co -datract.com -davidovic.info -dcj-nn.ru -delayreferat.ru +coslab.club +counterbot.com +customchocolate.business-for-home.com +dandiyabeats.in +darkbooks.org +day-news.info +dealwifi.com +dekoration.us delitime.info -deluxedumps.com +demenageur.com demon-tweeks.com derevesendeco.com -deshevo-nedorogo.ru -design-lands.ru -detskie-zabavi.ru -detsky-nabytek.info -deutschland123.de -dev-seo.blog -diesel-parts28.ru -digitalassetmanagement.site -diy-handmade-ideas.com -drev.biz -eaplay.ru -educatemetv.com +design-ideas.info +deutschehobbyhuren.net +discountbarbour.online +donna7753191.ru +donvito.unas.cz +downloads-whatsapp.com +driving.kiev.ua +dscaas.website +dverimegapolis.ru +e-kwiaciarz.pl +econom.co +ege-essay.ru +elektrischezigarette2.devhub.com elektrischezigarettekaufen2.cowblog.fr -elektrozigarette2.webs.com +elektrozigarette2.wordpressy.pl +elexies.info elitedollars.com -englate.com -epngo.bz -es5.com +enhand.se euromasterclass.ru european-torches.ru -event-tracking.com +euroskat.ru evrotekhservis.ru -extractorandburner.com -ezigarettenkaufen2.dreamwidth.org -ezigarettenshop2.postbit.com -face.hostingx.eu -faithe.top -falcon-images.blogspot.com -fetroshok.ru +excaliburfilms.com +exoclick.com +extlinks.com +extremez.net +ezigaretteshop.webs.com +fakehandbags.xyz +farm26.ru +fashionindeed.ml +fastfixing.tech +femdom.twiclub.in +fertilitetsradgivningen.se +fetishinside.com +fifa-coins.online +filkhbr.com filmetricsasia.com -filmidivx.com -firesub.pl -firstsiteguide.com -flex4launch.ru -floating-share-buttons.com -floridahuntingfishingadventures.com -flytourisme.org -forex.osobye.ru -fortevidyoze.net -forums.toucharcade.com -free-fb-traffic.com -freecamdollars.com -freewebs.com +financepoints.eu +findanysex.com +fireads.men +flashlarevista.com +floridamobilebillboards.com +footbalive.org +fordsonmajbor.cf +fortwosmartcar.pw +foto-sisek.porngalleries.top +fotoxxxru.com +fr.netlog.com +franecki.net +frankofficial.ru +freegamesplay.online +freeuploader.ml +frequiry.com +fres-news.com +freshmac.space +freshwallpapers.info froggytube.com -fsalas.com -galleryawesome.com -game-mmorpg.net -gazporno.com -get-seo-domain.com +fullfileaccess.com +gcup.ru +get-free-traffic-now.com getmiro.com -gg-arena.ru -gidonline.one -golden-catalog.pro -goldenggames.com -gombita.info -goodwinmetals.co -gopro-online.info -graphid.com -gsbs.com.ua -gsmtlf.ru -gta-top.ru -gwhwpxbw.bloger.index.hr -harmonyglen.us -hawaiielectriclight.com -highland-homes.com -hit-kino.ru -hoholikik.club -homemature.net -hotloans.ru -hotsocialz.com -hplaserjetpdriver8y.pen.io -hreade.com -hydropump.su -hyhj.info -idegenvezeto.eu -iklysha.ml -images-graphics-pics.com -imfamous.info +ghernnqr.skyrock.com +golmau.host.sk +goo.ne.jp +gosmeb.ru +gratuitxblcodes.com +greatgrace.ru +greatzip.com +grizzlysgrill.com +groupmoney.ru +gsasearchengineranker.xyz +gsasearchenginerankerdiscount.com +guildebzh.info +hdfreeporno.net +hdimagelib.com +hexpilot.com +host-tracker.com +hotelcrocenzi.sm +howopen.ru +humbmosquina.tk +ie.57883.net +ilbe.club +ilmexico.com +ilovevitaly.ru +imagerydatabase.com +impisr.edunsk.ru +impresagaia.it inboxdollars.com -interferencer.ru -investingclub.ru -investyb.com -invivo.hu -irkutsk.zrus.org -iscblog.info +infogame.name +infospot.pt +instasexyblog.com +intellekt21.ru +investpamm.ru +ip-guide.com +iron-age.info +isistaylorporn.info +islandminingsupply.wordpress.com ispac.org -itag.pw -itis4you.com +iwantedmoney.com +iweblist.info izismile.com -jackwolfskinoutlet.online -jav-idol.com -javcoast.com -johannesburgsingles.co.za -jongose.ninja -jwss.cc -kaac.ru -kakadu-interior.com.ua -kamagragelusa.net -kamorel.com -kazan.zrus.org -klin3952.ru -klosetkitten.com -krafte.ru -lanasshop.ru -landinez.co -landoftracking.com -larchik.net -law-check-nine.xyz -lenvred.org -linkrr.com -live-cam6.info -lollypopgaming.com -lonely-mature.com -lottospring.com -love-baby.cz -lumb.co -lunamedia.co -lyrster.com -m0r0zk0-krava.ru -macresource.co.uk -manualterap.roleforum.ru -marinetraffic.com -master-muznachas.ru -mastershef.club -matb3aa.com -meble-bogart.info -medicinacom.ru -meds-online24.com -megakino.net -microstatic.pl -millionare.com -minecraft-neo.ru -minecraft-rus.org -minet.club -mirobuvi.com.ua -missclub.info -mitrasound.ru -mnogabukaff.net -mobile-appster.ru +jav-fetish.site +jav-way.site +jbl-charge.info +jo24news.com +jobmarket.com.ua +jrcigars.com +jrpmakati.com +karadene.com +karting196.ru +kerwinandcariza.com +kihi.gdn +kinoflux.net +kiwi237au.tk +kladrus.ru +kleine-titten.biz +knowyournextmove.com +konkursov.net +krasivye-devushki.net +lacasamorett.com +lafriore.ru +lakomka.com.ua +lankarns.com +legionalpha.com +letto.by +lider-zhaluzi.kiev.ua +likesdesign.com +lineshops.biz +linkpulse.com +livingroomdecor.info +long-beach-air-conditioning.com +lost-alpha.ru +louboutinshoes.xyz +luckyshop.net.ua +mafa-free.com +magento-crew.net +makedo.ru +maladot.com +manve.info +markjaybeefractal.com +marmitaco.cat +mcnamaratech.com +mediafresh.online +medkletki.ru +meet-flirt-dating.com +megagrabber.ru +merryhouse.co.uk +mielec.pl +miniature.io +missvietnam.org mobile.ok.ru -mobioffertrck.com -modenamebel.ru -moesexy.com -mompussy.net -monclerjacketsoutlet.pw -morlat.altervista.org -morocco-nomad-excursions.com +mojpreskumanie.com +monclercoats.xyz +monetizer.com-01.site +moneyteam24.com +moscow-region.ru +moviezbonkerssk.cf moyakuhnia.ru -moyaterapiya.ru -mp3ringtone.info -mrlmedia.net -mrwhite.biz -mundoaberrante.com -mustat.com -my-cash-bot.co -mybinaryoptionsrobot.com -myfreecams.com -myfreetutorials.com -myghillie.info -myhydros.org -mylovelibrabry.com -myshopmatemac.com -namenectar.com -naphukete.ru -newsperuse.com -nicovideo.jp -nl.netlog.com -nrjmobile.fr -ntic.fr -nvformula.ru -o.light.d0t.ru -obesidadealgarve.com -oda.as -odywpjtw.bloger.index.hr -okmusic.jp -onlyforemont.ru -origin-my.ru -parajumpersoutlet.online -park.above.com -pay2me.pl -pensplan.com -photo.houseofgaga.ru -picscout.com +moz.com +mrbitsandbytes.com +musicktab.com +mymobilemoneypages.com +narkologiya-orel.ru +narkologiya-voronezh.ru +national-today-winning-winner.club +naval.jislaaik.com +news-speaker.com +nhl09.ru +ningessaybe.me +noumeda.com +nowtorrents.com +obiavo.in +obiavo.su +obsessionphrases.com +okayimage.com +onemantrip.com +onko-24.com +online247.ml +onlinetvseries.me +openmultipleurl.com +outclicks.net +outshop.ru +ownshop.win +oynat.info +oyster-green.com +paceform.com +partner-print.men +partner-stop.men +partnerworkroom.men +pekori.to pictures-and-images.net +picturesify.com +picturesmania.com +pinapchik.com pinkduck.ga -pinwallpaper.xyz -piter.xrus.org -pizda.lol -pochemychka.net -poisk-zakona.ru -popserve.adscpm.net -porndl.org -pornhub-ru.com +piuminiita.com +pk-pomosch.ru +ploenjitmedia.azurewebsites.net +pobeiranie.pl +pod-muzyku.club +popunder.ru porno-play.net -pornogad.com -pornohub.me -pornosee.info +porno.simple-image.com.ua +pornohd1080.online pornosok.ru -pornotubexxx.name +pornpost.in pornstartits.xblog.in -privatbank46.ru -proekt-mos.ru -profitkode.com -pronekut.com -propranolol40mg.blogspot.com -psoriasis-file.trade -radiodigital.co -ranksonic.org -razorweb-a.akamaihd.net -real-time-analytics.com -realtytimes.com -refudiatethissarah.info -regionshop.biz -registratciya-v-moskve.ru -registrationdomainsite.com -registrydomainservices.com -reining.lovasszovetseg.hu -remont-comp-pomosh.ru -renecaovillasale.online -rentalcarnavi.info +portside.cc +portside.xyz +potolokelekor.ru +pourvous.info +predmety.in.ua +professionaldieselcare.com +profolan.pl +promgirldresses.xyz +proposal-engine.com +prostitutki-yaroslavlya.org +proxyradar.com +pulse33.ru +quit-smoking.ga +qwesa.ru +rarbg.to +razleton.com +rednise.com +remmling.de repeatlogo.co.uk -responsive-test.net -restorator-msk.ru -resultshub-a.akamaihd.net -reversing.cc -rfd-split.hr -roleforum.ru -rucrypt.com -running-line.ru -russia-today-video.ru -russian-postindex.ru -rustic-quiver.win -ryetaw.com -s-iwantyou.com -sashagreyblog.ga -satellite.maps.ilovevitaly.com -scanner-elena.top -scenicmissouri.us -scrnet.biz.ua -sdelai-prosto.ru -se-welding.ru -search-error.com -searchkut.com -searchwebknow-a.akamaihd.net -semp.net -seocheckupx.com -seonetwizard.com -sexkontakte-seite.com -sexkvartal.com -sexsaoy.com -sexuria.net -shariki-zuma-lines.ru -shop-electron.ru -shoppingmiracles.co.uk -siteonomy.com -sk.golden-praga.ru -smartadserver.com -smartshoppymac.com -socialmadesimple.com -sodexo.com -soft-program.com -sogimlecal.tk -solicita.info -solitaire-game.ru -sovetogorod.ru -space2019.top -sprttrack.com -spy-sts.com -spyfu.com -spywareit.com +replicaclub.ru +responsinator.com +retreatia.com +robertefuller.com +rospromtest.ru +rp9.ru +ruf777.com +rukino.org +sabizonline.com +saddiechoua.com +sady-urala.ru +saveriopiazza.it +sbwealthsolutions.ca +scanmarine.info +scanner-fred.top +scanner-jack.top +search.1and1.com +secretscook.ru +seeingmeerkat.com +semprofile.com +seo-2-0.com +seoheap.com +seorankinglinks.us +seotoolsagency.com +sergiorossistore.online +servicecenter.co.ua +serving.adbetclickin.pink +sexstream.pl +sexystrippe.info +sfj-ror.no +simple-image.com.ua +simul.co +sitiz.club +sizeplus.work +slayerlife.com +social-buttons.com +social-search.me +softlinesolutions.me +softwaretrend.net +spabali.org +special-porn.com stairliftsarea.com +stardevine.com +startufa.ru starwars.wikia.com -stats-public.grammarly.io -stockquotes.wooeb.com -streamin.to +statustroll.com +stauga.altervista.org +steamoff.net +stefanbakosab.se +sticken.co +stockspmb.info +storehouse.ua +strana-krasoty.ru studworks.org -surfbuyermac.com -svolze.com -t-bygg.com -taaaak.com -teletype.in -templates.franklinfire.co -thelottosecrets.com -themestotal.com -thepokertimer.com -thesmartsearch.net -thetattoohut.com -timer4web.com -tizanidineingredients.blogspot.com -tizanidinevsvalium.blogspot.com +subj.ukr-lit.com +sundrugstore.com +supermesta.ru +superstarfloraluk.com +surgut.zrus.org +svtrd.com +taihouse.ru +taraz.xkaz.org +tedy.su +teenforporn.com +tehngr.ru +telefonsexi.com +telegramdownload10.com +template-kid.com +testingads.pro +thebitcoincode.com +theendivechronicles.com +thetardistimes.ovh +thewomenlife.com +thin.me.pn +tizanidineduringpregnancy.blogspot.com +tizanidineforopiatewithdrawal.blogspot.com +tizanidinepills.blogspot.com tmm-kurs.ru -totu.us -trafficjunky.com -trafficnetzwerk.de +tovaroboom.vast.ru +traffic2cash.org +trafficgenius.xyz tripper.de -trucri.me -truebeauty.cc -twu.com.ua -u-cheats.ru -u17795.netangels.ru -ukrtvir.com.ua +ttrraacckkrr.com +tubeline.biz +tula.mdverey.ru +twinderbella.com +tzritel.tk +ublaze.ru +udsgame.online +uhodzalijami.ru ukrup.com -ultimatesetnewfreeallsoftupgradesystems.pw -unionmarkt.de -unrealcommander.org -upstore.me -urlcut.ru -urldelivery.com -usiad.net -ussearche.cf -vacances-voyages.info -veerotech.com -venerologiya.com -venta-prom.ru -via-energy-acquistare.com -viagra.pp.ua -viddyoze.com -video-chat.in -video-production.com -videochat.mx -vigrx-original.ru +umekana.ru +uptime-as.net +urengoy.pro +url-img.link +usadacha.net +valoresito.com +vancleefreplica.pw +vasileostrovsky-rayon.ru +video-camer.com +videochat.world +videokrik.net +vietimgy.pw vikistars.com -vip2ch.com -vkontaktemusic.ru -vse-pesni.com -vykup-avto-krasnodar.ru -vzlom-na-zakaz.com -w7s.ru -wahicbefa31.soup.io -website-stealer.nufaq.com -westermarkanjou.se -westum.se -worldis.me +vinsit.ru +vip.51.la +vipsiterip.org +virtualbb.com +visitcambridge.org +vkmusics.ru +vroze.com +vsdshnik.com +vykupavto-krasnodar.ru +w3javascript.com +warezaccess.com +webix.me +websites-reviews.com +welcomeauto.ru +wetgames.ru +wingsofrefuge.net +wowas31.ucoz.ru +wowcasinoonline.ooo woweb.com.ua -wpsecurity.website -www.pinnacle-bets.com -xclicks.net +x-musics.com +xaylapdiendanang.com +xblog.in +xblognetwork.com +xgames-04.com +xjlottery.com xjrul.com -xkaz.org -xn-----7kceclhb4abre1b4a0ccl2fxch1a.xn--p1ai -xn----7sbho2agebbhlivy.xn--p1ai -xn--80aaafbn2bc2ahdfrfkln6l.xn--p1ai -xn--80aaagvmjabrs1aoc9luc.xn--p1ai -xn--80ahvj9e.xn--p1ai +xn-----6kcaacnblni5c5bicdpcmficy.xn--p1ai +xn----7sbifcamovvfggw9d.xn--p1ai +xn----8sbarihbihxpxqgaf0g1e.xn--80adxhks +xn----8sbdbjgb1ap7a9c4czbh.xn--p1acf +xn----8sbhefaln6acifdaon5c6f4axh.xn--p1ai +xn----btbdvdh4aafrfciljm6k.xn--p1ai +xn--80ajbshivpvn2i.xn--p1ai xn--80ak6aa92e.com -xn--90acenikpebbdd4f6d.xn--p1ai +xn--b1addnj3cah.xn--p1ai +xn--b1ag5cfn.xn--p1ai +xn--e1aggki3c.xn--80adxhks xnxx699.com -xtraffic.plus -xtubeporno.net -yachts-cruise.info -yeartwit.com -yellowproxy.net -youradexchange.com -youradulthosting.com -youtubedownload.org -zajm-zalog-krasnodar.ru -zakonobosago.ru +xsion.net +xuki.us +xxxhdvideo.site +youdao.com +youjizz.com +youporn-ru.com +youresponsive.com +yunque.pluto.ro +zaapplesales.blogspot.com +zambini.ru +zarenica.net +zeroredirect1.com zeroredirect12.com -zeroredirect2.com -zeroredirect7.com -zolotoy-lis.ru -zoogdisany.com +zhacker.net +zhuravlev.info +zionstar.net +zmoda.hostreo.com +zona-aqua.ru +zoogdiesney.com zrelaya.pw diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index 7e9ff8749..09553032e 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -1001desert.com -13tabs.com +101lesbian.xyz +101raccoon.ru +108shot.com +1adult.com 214jaluwobapef.cf -3rbseyes.com -83net.jp -8gold.com -adinterax.com -adsfresh.men -advertisingtag.net -ageofclones.com -aludecor.info -animalrank.com -apisecretsaucebi-a.akamaihd.net -appfixing.space -autotrends.today -azazaporn.com -azbukafree.com -baksman.com -bankofthewext.com -belinka.com.ua -bestcurs.org -bestsexyblog.com -bestssaker.com -blackwitchcraft.ru -blogrankers.com -bonzibuddi.com -brandedleadgeneration.com -budpost.com.ua -buehne-fuer-menschenrechte.de -bumskontakte.org -c1.onedmp.com -candelluxsklep.pl -carapics.com -cardul.ru -cartierjusteunclou.xyz -cartierloveringreplica.xyz -cascadelink.org -cb.iphantom.com -cercacamion.it -com-cleaner.systems -com-onlinesupport.site -compliance-elena.xyz -compliance-mark.top -contentlook.co -cookielawblog.wordpress.com -crackzplanet.com -cv.wallhade.co -dealitright.click -dentalpearls.com.au -detmebel.su -dgfitness.co -docsportal.net -doktoronline.no -downloaddy.net -drivotracker.com -droidlook.net -druzhbany.ru -druzhininevgeniy63.blogspot.com +51unlim.ru +777-club.ru +8si.ru +activepr.ru +acunetix-referrer.com +adidas.frwebs.fr +adsref.men +adultfullhd.com +advancedmassagebysara.com +affordablewebsitesandmobileapps.com +akusherok.ru +allfinweb.com +alltheviews.com +alvtank.se +amazon-seo-service.com +amt-k.ru +anapa-inns.ru +antons-transporte.de +apispringsmartne-a.akamaihd.net +apps-analytics.net +asscenihotosidea.blogspot.co.za +audiobangout.com +automobile-spec.com +avek.ru +azartmix.com +banki76.ru +battle.net +beachtoday.ru +beauty-bracelet.com +bestwebsitesawards.com +beztuberkuleza.ru +bibys.com +bizcheapjerseyswholesalechina.com +blagovest-med.ru +braindaily.xyz +brandimensions.com +bristolhotel.com.ua +bvlgaribracelet.xyz +callawaygolfstore.online +canzoni.ru +capacitacionyaprendizaje.com +cashonads.com +cdncash.com +cenokos.ru +club-samodelkin.ru +co.lumb.co +cometorussia.net +cookie-law-enforcement-ff.xyz +custodieva.ru +cyse.tk +dafatiri.com +davidovic.info +dealighted.com +dickssportinggoods.com +dikqlyremy.info +discovertreasure-a.akamaihd.net +downloads-whatsapp.com +drugstoreforyou.com +e-collantes.com e-commerce-seo1.com -ekaterinburg.xrus.org -eloxal.ru -erot.co -errorfixing.space -euroskat.ru -expertblog.info -extlabs.io -ezaz.info -fanpagerobot.com -fashionavenuegame.com -ferrotodo.com -fetlifeblog.com -filesclub.net -filesvine.com -filmgo.ru -filmidivx.com -financeloan.us -firstdrugmall.ru -flavors.me -fortunejack.com -fotopop.club -free-today.com -front.ru -funtoonez.com -gamedayhouse.com -gate5.co.za -gfaq.ru -glopages.ru -googlsucks.com -greatfind-a.akamaihd.net -gsou.cf -gwhwpxbw.bloger.index.hr -h2monline.com -hacktougroup.ru -hahashka.ru -hasfun.com -healgastro.com -heliko.no -hoverboard360.de -ibb.com.ua -ilikevitaly.com -imagefinder.site -imagine-ex.co -imgata.com -imitex-plus.ru -incomekey.net -infobabki.ru -investsuccess.org -investyb.com -iradiology.ru -ispac.org -jabimgo.pw +egovaleo.it +eldiariodeguadalajara.com +elkacentr.ru +englishdictionaryfree.com +everytalk.tv +extstat.com +ezigarettekaufen2.mpbloggar.se +farmprofi.net +fashion-mk.net +fatfasts-4tmz.com +fbdownloader.com +fermersovet.ru +flowwwers.com +flytourisme.org +fm-upgrade.ru +fortevidyoze.net +forum.poker4life.ru +free-fb-traffic.com +freeuploader.ml +gael-s.ru +gamezblox.com +gay.adultgalls.com +gg.zzyjxs.com +glavtral.ru +gq-catalog.gq +gsbs.com.ua +gstatey.net +guitar-master.org +habermetre.com +hdimagegallery.net +hdxnxxtube.mobi +herokuapp.com +himazin.info +homedo.fabpage.com +hundejo.com +idc.com.ua +importchinacoach-teach.com +infogame.name +insomniagamingfestival.com +internetartfair.com +ipostroika.ru +irunfar.com james13prix.info -kazan.zrus.org +jerseysbizwholesalecheap.com +joy-penguin.com +jrcigars.com +kashubadesign.ru keywordglobal.co.za -kidskunst.info -kinomaniatv.pl -kiwi237au.tk -kletkimehan.ru -kolotiloff.ru -krafte.ru -ledpolice.ru -littleberry.ru -lkbennettstore.online -lodki-pvh.dp.ua -lunamedia.co -m-google.xyz -mabdoola.blogspot.com -masturbate.co.uk -maximpartnerspr.com -mebelcomplekt.ru -miniads.ca -mister-shop.com -mmoguider.ru -mobilemedia.md -moinozhki.com -moneymaster.ru -moneyzzz.ru -montredemarque.nl -mowser.com -mydirtystuff.com -mywallpaper.top -narkologiya-penza.ru -net-radar.com -nordvpn.com -nosecret.com.ua -o-dachnik.ru -o-o-8-o-o.ru -obiavo.ru -ooo-gotovie.ru -paclitor.com -paradontozanet.ru -partybunny.ru -pekori.to -pix24x7.com -pladform.ru -polybuild.ru -porno-transsexuals.ru -powenlite24.ru -predominant-invent.tk -privatefx-in.ru -prizesbook.online -progonrumarket.ru -progress-upakovka.ru +keywordsuggest.org +kit-opt.ru +kol-energo.ru +ktotut.net +labvis.host.sk +lacloop.info +lanasshop.ru +lancheck.net +laudit.ru +levaquin750mg.blogspot.com +lifehacĸer.com +limtu.ifmo.ru +lincolntheatre.com +live-sexcam.tk +livingroomdecor.info +maridan.com.ua +matb3aa.com +megaindex.ru +michaelkorsoutlet.store +minecraft-neo.ru +missis.top +mixx.com +mnogabukaff.net +moncleroutletonline.win +mountainstream.ms +mttwtrack.com +mygameplus.com +myliveblog.ru +nash-krym.info +naturalshair.site +neks.info +netanalytics.xyz +nikitsyringedrivelg.pen.io +nucia.biz.ly +obsessionphrases.com +ohmyrings.com +online247.ml +oralsexfilme.net +orenburg-gsm.ru +paccohichetoti.ml +partnersafe.men +pcboa.se +perfectpracticeweb.com +pics-group.com +pictures-and-images.com +pinstake.com +popunder.ru +porno-play.net +pornophoto.xyz +prizefestival.mobi +pro-tec.kz +profolan.pl prokotov.com -prostitutki-yaroslavlya.org -pst2017.onlinewebshop.net -puzzleweb.ru -quickbuck.com -quickcashlimited.com -r-e-f-e-r-e-r.com -razyboard.com -rcb101.ru -realtytimes.com -redhotfreebies.co.uk -revistaindustria.com -ritlweb.com -rukino.org -sarf3omlat.com -sayyoethe.blogspot.co.za -sbtdesign.co.uk -scanner-andrew.top -scottbywater.com -scrapinghub.com -searchkut.com -seo-2-0.com -seocheckupx.com -seocheki.net -service-core.ru -sex-znakomstva.online -sexflirtbook.com -sexkontakteao.info -shinikiev.com.ua -siha.de -simpoed.ufop.br -sisi-go.ru -siteheart.net -sizeplus.work -soc-econom-problems.ru -soft-terminal.ru -spaceshipad.com -spb.afora.ru -spin2016.cf -sport7777.net -sprttrack.com -spywarebegone.com -streetfooduncovered.com -stroymonolit.su -styro.ru -sugarkun.com -supermesta.ru -tds-advert005.info -telegraf.by -theguardlan.com -theheroes.ru -thin.me.pn -tizanidine4mgstreetvalue.blogspot.com -tizanidinehcl2mgsideeffects.blogspot.com -tonivedu.it -topclickguru.com -toursmaps.com -track112.site -traffic2money.com -trafficfactory.biz -treasuretrack-a.akamaihd.net -ukrtextbook.com -unrealcommander.com -urengoy.pro -urlopener.com -usswrite.com -verymes.xyz +promalp-universal.ru +prostitutki-almata.org +purchasepillsnorx.com +pushdata.sendpulse.com +radiodigital.co +real-time-analytics.com +remontgruzovik.ru +rff-cfal.info +rixpix.ru +sashagreyblog.ga +saveindex.xyz +sawin.beth.webd.pl +scanner-donald.top +scanner-fred.top +scanner-jack.top +scanner-margo.top +scanner-mary.top +scat.porn +searchimpression.com +searchwebknow-a.akamaihd.net +security60-e.com +selectads.men +seo18.su +seobility.net +seofirmreviewsus.info +seojokes.net +sex-tracker.com +sexkontakte-seite.com +sexvideo-sex.com +simple-share-buttons.com +skatestick.bid +sluganarodu.ru +social-buttons.xyz +sockshares.tv +soda.media +solution4u.com +spywarespy.com +streamin.to +surfbuyermac.com +swinon.site +symphonyintegratedhealthcare.com +telemetryverification.net +texbaza.by +theprofitsmaker.net +thetattoohut.com +titangel-vietnam.com +tizanidinedrugtest.blogspot.com +tizanidineinfo.blogspot.com +tizanidinesleepaid.blogspot.com +tizanidinestreetprice.blogspot.com +tizanidinezanaflex.blogspot.com +top-deal.com.pl +top-study.work +top10-way.com +track2.shop +tracxn.com +truck-rece.life +tube8.com +uhodzalijami.ru +vchulkax.com veselokloun.ru -video-woman.com -vikistars.com -vzlomfb.com -web.cvut.cz -webmasters.stackexchange.com +viktoria-center.ru +voluumtracker1.com +vzlomsn.org +vzlomtw.com +wallpapersdesk.info +warezkeeper.com +webcamtalk.net +webradiology.ru +website-speed-up.top weddingdresses.xyz -wieseversa.no -wildworld.site -wiosenny-bon-1500.pl -workle.website -wtsindia.in -www.wohnkabinen-shop.de -xgames-04.com -xn--l1aengat.xn--p1ai -yginekologa.com -youradexchange.com +x-porno.video +xingzi-vision.com +xn--80aeahghtf8ac5i.xn--p1ai +xnxxandxvideos.com +xxlargepop.com +xxxtubesafari.com +youjizz.com +youtube-downloader.savetubevideo.com +zeroredirect6.com +zog.link zoodrawings.com -zx6.ru -zygophyceous.womanstars.site +zooggames.com +zoolubimets.ru diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index 0774f55a5..981829a62 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.131 0; +10.1.0.90 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index 10503a2fb..b0bf12aa1 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -21214,3 +21214,1251 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2468 +### Updated: Mon Jul 12 14:05:20 UTC 2021 +### Bad Bot Count: 615 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: OPPO A33 +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ + diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 10503a2fb..b0bf12aa1 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -21214,3 +21214,1251 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2468 +### Updated: Mon Jul 12 14:05:20 UTC 2021 +### Bad Bot Count: 615 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: OPPO A33 +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ +