From 0e8f48cef14a4a466678003db5b4ab74a209107f Mon Sep 17 00:00:00 2001 From: mitchellkrogza Date: Sun, 11 Jul 2021 14:10:54 +0000 Subject: [PATCH] V4.2021.07.2467 [ci skip] --- AUTO-CONFIGURATION.md | 2 +- MANUAL-CONFIGURATION.md | 2 +- README.md | 2 +- _google_analytics_ghost_spam/README.md | 2 +- .../etc/nginx/conf.d/globalblacklist.conf | 4 +- conf.d/globalblacklist-testing-version.conf | 4 +- conf.d/globalblacklist.conf | 4 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- dev-tools/buildnumber | 2 +- dev-tools/globalblacklist-testing.template | 4 +- dev-tools/globalblacklist.template | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- dev-tools/test_units/blacklist-ips.conf | 2 +- .../random-bots-for-test-quick.list | 20 +- .../test_units/random-bots-for-test.list | 290 ++-- .../random-bots-for-whitelist-test.list | 296 ++-- .../random-referrers-for-test-quick.list | 20 +- .../test_units/random-referrers-for-test.list | 934 ++++++------ .../random-referrers-for-whitelist-test.list | 474 +++---- dev-tools/test_units/ratelimittest-beta.txt | 14 +- dev-tools/test_units/ratelimittest-master.txt | 10 + dev-tools/test_units/whitelist-ips.conf | 2 +- dev-tools/tmp/robots.txt | 1248 +++++++++++++++++ robots.txt/robots.txt | 1248 +++++++++++++++++ 47 files changed, 3587 insertions(+), 1071 deletions(-) diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 20bc5230e..0b867b2f4 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2466 +#### Version: V4.2021.07.2467 #### Bad Referrers Blocked: [7067](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 0a4db4315..b3f4bb35c 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2466 +#### Version: V4.2021.07.2467 #### Bad Referrers Blocked: [7067](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/README.md b/README.md index 8f362b7b0..f47427fad 100755 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2021.07.2466 +#### Version: V4.2021.07.2467 #### Bad Referrers Blocked: [7067](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 884674bbd..801002569 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,7 +3,7 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2021.07.2466 +#### Version: V4.2021.07.2467 #### Bad Referrers Blocked: [7067](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index b57b42fad..f2019c1a4 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2466 -### Updated: Sat Jul 10 14:05:04 UTC 2021 +### Version: V4.2021.07.2467 +### Updated: Sun Jul 11 14:05:07 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index c2018afb4..706738c14 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2466 -### Updated: Sat Jul 10 14:05:05 UTC 2021 +### Version: V3.2021.07.2467 +### Updated: Sun Jul 11 14:05:08 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index b57b42fad..f2019c1a4 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2466 -### Updated: Sat Jul 10 14:05:04 UTC 2021 +### Version: V4.2021.07.2467 +### Updated: Sun Jul 11 14:05:07 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf index c2018afb4..706738c14 100644 --- a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2466 -### Updated: Sat Jul 10 14:05:05 UTC 2021 +### Version: V3.2021.07.2467 +### Updated: Sun Jul 11 14:05:08 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index fcbf7c951..ca0f2829d 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.236 1; +10.1.0.131 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 8fa087902..0774f55a5 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.236 0; +10.1.0.131 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index c2018afb4..706738c14 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2466 -### Updated: Sat Jul 10 14:05:05 UTC 2021 +### Version: V3.2021.07.2467 +### Updated: Sun Jul 11 14:05:08 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index fcbf7c951..ca0f2829d 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.236 1; +10.1.0.131 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 8fa087902..0774f55a5 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.236 0; +10.1.0.131 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index c2018afb4..706738c14 100644 --- a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2466 -### Updated: Sat Jul 10 14:05:05 UTC 2021 +### Version: V3.2021.07.2467 +### Updated: Sun Jul 11 14:05:08 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index fcbf7c951..ca0f2829d 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.236 1; +10.1.0.131 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 8fa087902..0774f55a5 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.236 0; +10.1.0.131 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index c2018afb4..706738c14 100644 --- a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2466 -### Updated: Sat Jul 10 14:05:05 UTC 2021 +### Version: V3.2021.07.2467 +### Updated: Sun Jul 11 14:05:08 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index 8da928e69..92a41bad6 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -2466 +2467 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index c2018afb4..706738c14 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2466 -### Updated: Sat Jul 10 14:05:05 UTC 2021 +### Version: V3.2021.07.2467 +### Updated: Sun Jul 11 14:05:08 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index b57b42fad..f2019c1a4 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2466 -### Updated: Sat Jul 10 14:05:04 UTC 2021 +### Version: V4.2021.07.2467 +### Updated: Sun Jul 11 14:05:07 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index ce12bb129..d55a6b844 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -52.175.239.81 0; +20.102.120.177 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index b57b42fad..f2019c1a4 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2466 -### Updated: Sat Jul 10 14:05:04 UTC 2021 +### Version: V4.2021.07.2467 +### Updated: Sun Jul 11 14:05:07 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index fcbf7c951..ca0f2829d 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.236 1; +10.1.0.131 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 8fa087902..0774f55a5 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.236 0; +10.1.0.131 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index b57b42fad..f2019c1a4 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2466 -### Updated: Sat Jul 10 14:05:04 UTC 2021 +### Version: V4.2021.07.2467 +### Updated: Sun Jul 11 14:05:07 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index fcbf7c951..ca0f2829d 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.236 1; +10.1.0.131 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 8fa087902..0774f55a5 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.236 0; +10.1.0.131 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index b57b42fad..f2019c1a4 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2466 -### Updated: Sat Jul 10 14:05:04 UTC 2021 +### Version: V4.2021.07.2467 +### Updated: Sun Jul 11 14:05:07 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index fcbf7c951..ca0f2829d 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.236 1; +10.1.0.131 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 8fa087902..0774f55a5 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.236 0; +10.1.0.131 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index b57b42fad..f2019c1a4 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2466 -### Updated: Sat Jul 10 14:05:04 UTC 2021 +### Version: V4.2021.07.2467 +### Updated: Sun Jul 11 14:05:07 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index ce12bb129..d55a6b844 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -52.175.239.81 0; +20.102.120.177 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 3bff63152..85d240efb 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2466 -### Updated: Sat Jul 10 14:05:04 UTC 2021 +### Version: V4.2021.07.2467 +### Updated: Sun Jul 11 14:05:07 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index ce12bb129..d55a6b844 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -52.175.239.81 0; +20.102.120.177 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 8ad27ada6..2121275ca 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2466 -### Updated: Sat Jul 10 14:05:04 UTC 2021 +### Version: V4.2021.07.2467 +### Updated: Sun Jul 11 14:05:07 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index fcbf7c951..ca0f2829d 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.236 1; +10.1.0.131 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 06a529b0d..35946800d 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Bitacle -Brandwatch -MTRobot -Morfeus Fucking Scanner -Netvibes -Probethenet -SBIder -SEOprofiler -SuperBot -Web Sauger +DomCopBot +GrapeFX +IRLbot +PageScorer +Ripper +WEBDAV +Wallpapers/3.0 +Web Fuck +domainsproject.org +sexsearcher diff --git a/dev-tools/test_units/random-bots-for-test.list b/dev-tools/test_units/random-bots-for-test.list index 1b84ce1b7..38f33ae3f 100644 --- a/dev-tools/test_units/random-bots-for-test.list +++ b/dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -360Spider -404checker -404enemy +01h4x.com +80legs +ADmantX Abonti -AfD-Verbotsverfahren -AhrefsBot +Acunetix AiHitBot -Alligator -AlphaBot -Anarchy99 -Anthill +Aipbot +Alexibot +Ankit Apexoo +Aspiegel +Asterias AwarioRssBot -BDCbot +AwarioSmartBot BackWeb +BacklinkCrawler Badass -Barkrowler -Battleztar Bazinga +Bandit +BatchFTP Bitacle -BlowFish -Bolt -Buddy +Boardreader +Buck BuiltBotTough BuiltWith Bullseye -CATExplorador +BunnySlippers +BuzzSumo Calculon +CazoodleBot Cegbfeieh -CheeseBot +CensysInspect ChinaClaw -Chlooe -Cliqzbot -Cogentbot Collector CopyRightCheck Cosmos -Craftbot -CrazyWebCrawler -CrunchBot +Crescent +Curious +Custo CyotekWebCopy +DBLBot DSearch DTS Agent -Deusu -Devil -Digincore +DataCha0s +DatabaseDriverMysqli Disco +Discobot Discoverybot -DnyzBot -DomainCrawler +DittoSpyder +DomCopBot +DomainAppender DomainStatsBot -Domains Project -Dotbot -Dragonfly -Drip -ECCP/1.0 -EMail Wolf -Ebingbong -EroCrawler +Download Wonder Express WebPictures -ExtractorPro +ExtLinksBot +Extractor Extreme Picture Finder EyeNetIE -FemtosearchBot +Ezooms Fimap -FrontPage -Fuzz +Firefox/7.0 +Foobot +Fyrebot G-i-g-a-b-o-t -GetRight +GT::WWW +GermCrawler Gigablast +Go!Zilla +Go-Ahead-Got-It GoZilla -GrabNet Grabber -Grafula GrapeFX -HMView +HTTP::Lite +HTTrack Haansoft HaosouSpider +Harvest Havij +Hloader IDBTE4M -IRLbot Iblog -Indy Library -Intelliseek -Iria -Iskanie +Id-search +IlseBot +Image Fetch +JOC Web Spider JennyBot -JetCar Jetty JustView Kenjin Spider -Keyword Density -Kinza -LWP::Simple -Lanshanbot +Kozmosbot +LNSpiderguy +Larbin +LeechGet LexiBot -Libwhisker -Likse +Lftp +LieBaoFast +Lightspeedsystems +LinkWalker Linkbot -LinkpadBot -Lipperhey -Lmspider +LinksManager +LinqiaMetadataDownloaderBot +LinqiaRSSBot MIDown tool -MIIxpc -Mail.RU_Bot +MQQBrowser +Magnet +Majestic12 MarkMonitor Mata Hari +MauiBot +Mb2345Browser Mediatoolkitbot Metauri Mojeek +Mojolicious Morfeus Fucking Scanner -Mozlila -Mr.4x3 -Msrabot -NICErsPRO -NPbot -Name Intelligence +Musobot +Nameprotect Navroad NearSite -Net Vampire +Needle +NetAnts +NetLyzer NetMechanic -NetSpider +NetZIP +Nettrack +NextGenSearchBot Nibbler -Niki-bot -Nikto -Ninja Nmap Nutch -OPPO A33 Octopus -Offline Navigator +Offline Explorer OnCrawl -PECL::HTTP -PageGrabber +OpenLinkProfiler +Openfind +Openvas +OrangeSpider +OutfoxBot +PHPCrawl +POE-Component-Client-HTTP +Page Analyzer +PageAnalyzer PageScorer -Pandalytics +Panscient Papa Foto -Pavuk Petalbot -Picsearch +Pi-Monster Piepmatz -Pixray -Pockey -ProPowerBot +PleaseCrawl ProWebWalker -Psbot Pu_iN -Pump QueryN Metasearch RankFlex RankingBot2 Rankivabot ReGet +Reaper +RebelMouse Recorder -RedesScrapy -Rogerbot +SEOkicks-Robot SEOlyticsCrawler -SISTRIX -SalesIntelligent +SEOstats +SMTBot ScanAlert -Scrapy +ScrepyBot Searchestate -SearchmetricsBot -SentiBot -SeoSiteCheckup +SemrushBot Shodan -SiteCheckerBotCrawler -SiteExplorer +Site Sucker +SiteLockSpider +Sitebeam Sitevigil -Snake +SmartDownload +Snoopy SocialRankIOBot -Sociscraper -Sogou web spider -Sosospider Sottopop -Spammen -Spinn3r -SputnikBot -Sqlmap -Sqworm +SpankBot +Spanner +Spbot Steeler Stripper -Sucker +Sucuri +SuperBot SuperHTTP -Surfbot -SurveyBot -T0PHackTeam +Suzuran +Swiftbot T8Abot -Telesphorep -The Intraformant -Titan +TeleportPro +Telesoft +TheNomad Toweyabot -Trendiction -Trendictionbot -TwengaBot -Twice +True_Robot +Turnitin +TurnitinBot +URLy Warning +UnisterBot +Upflow +VCI +Vacuum Vagabondo -WBSearchBot -WEBDAV -WISENutbot -WPScan -WWW-Mechanize -WWW::Mechanize +Virusdie +VoidEYE +WASALive-Bot +WallpapersHD WeSEE -Web Pix +Web Collage +Web Enhancer Web Sauger -Web Sucker WebCollage -WebEnhancer -WebFuck +WebFetch +WebGo IS +WebImageCollector +WebLeacher +WebReaper +WebSauger WebStripper -WebSucker +WebZIP +Webalta Website Quester +Whacker +Whatweb Who.is Bot Widow +WinHTTrack WiseGuys Robot -Xaldon WebSpider Xaldon_WebSpider -Xenu -Zauba -Zermelo -ZyBorg -archive.org_bot -arquivo-web-crawler -backlink-check -check1.exe -com.plumanalytics +Zade +ZoomBot crawl.sogou.com -crawler4j +demandbase-bot eCatch evc-batch gopher +heritrix internetVista monitor isitwp.com -magpie-crawler -meanpathbot -mediawords -netEstate NE Crawler +page scorer pcBrowser +plumanalytics +polaris version +probe-image-size ripz satoristudio.net +scalaj-http scan.lol -seostar -serpstatbot -sexsearcher +seoscanners siteripz sogouspider -spyfu -sysscan +trendiction.com trendiction.de +voyagerx.com xpymep1.exe diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 2756d2dd1..9c8f9a1ac 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ +01h4x.com +360Spider +404checker 404enemy -80legs ADmantX -AIBOT -ASPSeek Abonti -Aboundexbot AfD-Verbotsverfahren +AhrefsBot AiHitBot +Alexibot AllSubmitter -AlphaBot -Anthill -BBBike +Anarchie +Apexoo +BDFetch BLEXBot -BackDoorBot BackStreet -BackWeb -Barkrowler -Battleztar Bazinga -Black Hole +Backlink-Ceck +Badass +Bandit +BetaBot +Bigfoot BlowFish -Boardreader -Bolt +BotALot Brandprotect +BuiltWith +BunnySlippers BuzzSumo -CATExplorador -CCBot -Calculon -Claritybot +Cegbfeieh +Cloud mapping +Cogentbot Collector -CrazyWebCrawler -Crescent -Custo +CopyRightCheck +Copyscape +Curious CyotekWebCopy DIIbot -DSearch -DigitalPebble +DataCha0s +DatabaseDriverMysqli +Deusu Dirbuster -Disco Discoverybot +DittoSpyder DnyzBot -DomainAppender -DomainCrawler +DomainSigmaCrawler +DomainStatsBot Domains Project Dotbot +Drip ECCP/1.0 -EMail Siphon -EirGrabber -EroCrawler -Evil -Exabot +EMail Wolf +Ecxi ExtractorPro -EyeNetIE +Extreme Picture Finder +Ezooms +FDM FHscan -Foobot +Fimap +FlashGet +Flunky Freeuploader FrontPage -Fuzz -FyberSpider +Fyrebot +G-i-g-a-b-o-t +GT::WWW +GermCrawler +GetRight +Getintent +Gigablast +Gigabot +Gotit GrabNet -GridBot +Grafula +GrapeFX +GrapeshotCrawler HMView +HTMLparser HTTP::Lite -Haansoft HaosouSpider -Havij -Humanlinks -IDBTE4M +Heritrix +Hloader +HybridBot +IRLbot +Iblog IlseBot -Image Sucker -IndeedBot Indy Library +InfoNaviRobot +InfoTekies +Internet Ninja +InternetSeer +IstellaBot +JOC Web Spider +JamesBOT +Jbrofuzz +JennyBot +JetCar Joomla -JustView -Keyword Density -LWP::Simple +Kenjin Spider Lanshanbot LeechFTP +LeechGet +LexiBot Lftp -Libwhisker +LibWeb LieBaoFast Lightspeedsystems -LinkScan -LinksManager -LinqiaRSSBot +Likse +LinkWalker +Linkbot +Linkdexbot +LinkextractorPro +Lipperhey Lipperhey Spider Litemage_walker -Ltx71 -MFC_Tear_Sample -MQQBrowser -Mag-Net +Lmspider +MSFrontPage +MTRobot Mail.RU_Bot -Majestic-SEO -Majestic12 +Majestic SEO Masscan -MauiBot +Mata Hari Mb2345Browser -MeanPath Bot +Meanpathbot +Mediatoolkitbot +MegaIndex.ru +MicroMessenger +Microsoft Data Access +Microsoft URL Control Mister PiX Moblie Safari Mojeek Mozlila -Mr.4x3 -Navroad -Needle -Nessus -NetLyzer -NetSpider +Musobot +NetAnts +NetMechanic NetZIP -Netcraft -Nettrack Netvibes -Nibbler -Niki-bot -NimbleCrawler +NextGenSearchBot +Nimbostratus Nmap -Nuclei -Octopus -OnCrawl OpenLinkProfiler Openfind -OrangeBot -OutclicksBot -Page Analyzer +OrangeSpider +PECL::HTTP +POE-Component-Client-HTTP PageGrabber PageScorer -PageThing.com -Panscient PeoplePal +Picscout Picsearch +PictureFinder Pimonster -Pockey -Probethenet -Quick-Crawler -RSSingBot -RankActiveLinkBot +ProWebWalker +Psbot +Pump +PyCurl RankFlex -RankurBot +RankingBot ReGet -Recorder -RedesScrapy -RepoMonkey -RocketCrawler -SEOkicks -SEOkicks-Robot +Rogerbot SEOlyticsCrawler +SEOprofiler SISTRIX -SalesIntelligent -ScanAlert -ScoutJet +Scanbot Scrapy -Screaming -ScreenerBot -Searchestate -SemrushBot +SearchmetricsBot +Semrush SeoSiteCheckup -SeobilityBot -Shodan Siphon SiteCheckerBotCrawler SiteLockSpider SiteSucker +Sitebeam +Siteimprove Sitevigil -SmartDownload -Snapbot +SlySearch +Snake Snoopy +SocialRankIOBot Sociscraper -SpaceBison -Spammen -Spanner -Spinn3r -Sqlworm -Sqworm +SpankBot +Spbot Steeler -Stripper -SuperBot -Suzuran +Sucker T0PHackTeam -Teleport TeleportPro -Tracemyfile -Twice +Telesphorep +The Intraformant +TheNomad +Toata +Trendiction +Trendictionbot +True_Robot +Turingos URLy Warning URLy.Warning UnisterBot +Upflow +V-BOT VCI -Vagabondo VelenPublicWebCrawler -VeriCiteCrawler VidibleScraper +Voil Voltron -WASALive-Bot WBSearchBot +WPScan WWW-Collector-E WWWOFFLE -Wallpapers/3.0 -WeSEE -Web Auto Web Enhancer -Web Fetch Web Pix Web Sauger Web Sucker -WebBandit -WebEnhancer -WebFuck -WebGo IS +WebCopier +WebFetch WebLeacher -WebReaper -WebSauger +WebPix +WebWhacker WebZIP -WebmasterWorldForumBot WebsiteExtractor +WebsiteQuester +Webster +Whack Widow WinHTTrack -Wonderbot +Woobot Wprecon -Xaldon WebSpider -YoudaoBot -Zade Zauba -Zitebot ZmEu -ZoomBot -ZumBot -arquivo-web-crawler +ZyBorg backlink-check -com.plumanalytics -demandbase-bot -domainsproject.org +check1.exe +evc-batch facebookscraper -gopher -heritrix -instabid -internetVista monitor ips-agent -lwp-request -magpie-crawler +mediawords +netEstate NE Crawler oBot -page scorer -plumanalytics -probe-image-size -scan.lol -seobility -seostar +polaris version +ripz +s1z.ru +seoscanners +sitechecker.pro sogouspider +sp_auditbot tAkeOut +trendiction.com webmeup-crawler x09Mozilla x22Mozilla +zauba.io +zgrab diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index bd119c809..259e0e536 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -api.stathat.com -blobar.org -gemgrab-a.akamaihd.net -hotel-mkad.ru -picturesboss.com -rumamba.com -secret.xn--oogle-wmc.com -telsis.com -valueclick.com -xedserver.com +1qingdao.com +arewater.com +autoloans.com +culad.com +dollartree.info +gate5.co.za +greenshop.su +krasivoe-hd.com +lookingglassemb.com +vedomstvo.net diff --git a/dev-tools/test_units/random-referrers-for-test.list b/dev-tools/test_units/random-referrers-for-test.list index 782795e1a..b5ddecdfc 100644 --- a/dev-tools/test_units/random-referrers-for-test.list +++ b/dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -0912701309f8ce.com -123cha.com -1688.com -18ps.ru -1kdailyprofit.me -2girls1cup-free.com -300richmond.co.nz -3weekdiet.com -8gold.com -aadroid.net +000free.us +1008.su +1webmaster.ml +2015god.org +2ads.co.uk +4istoshop.com +4pp13.com +5u.com +72-news.com +7minuteworkout.com +aa08daf7e13b6345e09e92f771507fa5f4.com +aa14ab57a3339c4064bd9ae6fad7495b5f.com +aa9d046aab36af4ff182f097f840430d51.com +aaa38852e886ac4af1a3cff9b47cab6272.com abcdefh.xyz -abcdeg.xyz -adtech.us +actices.com +acunetix-referrer.com +adbetclickin.pink +adelly.bg +admanaerofoil.com +admeasures.com +adnotbad.com advancedcleaner.com -aero2.ru -aghanyna.com -albuteroli.com -alfapro.ru -alienwheel.es -alldownload.pw -allornamenti.com -allsilver925.co.il -alt-servis.ru -amateurgalls.com -amigobulls.com -androids-store.com -apiadanaknet-a.akamaihd.net -approvedlocal.co.za -arenanews.com.ua -ascat.porn -asiavirtualsolutions.com -asrv-a.akamaihd.net -atlasvkusov.ru -azarttoto.com -azazu.ru -babieca.com -babosas.com -balans.shahterworld.org -bankcrediti.ru -bavsac.com -beremenyashka.com -bestempresas.es -bestmarriages.com +adventureparkcostarica.com +aerogo.com +afslankpillen2017nl.eu +agreda.pluto.ro +ahrefs.com +aihelen.net +albumsuper.info +alfa9.com +amazingninja.com +amazingpic.net +ameblo.jp +ampower.me +anonymizeme.pro +aproposde.com +arabgirls.adultgalls.com +arabsexxxtube.com +arabseyes.com +arenda-yeisk.ru +arpelsreplica.xyz +aruplighting.com +arvut.org +asrv-a.akamoihd.net +astana.xxxkaz.org +atovh.local-finders.com +autonew.biz +aviapanda.ru +aviav.co +avirasecureserver.com +avitocars.ru +avon-novosib.ru +b-styles.xyz +babyguns.ru +backiee.com +banksrf.ru +beatifullandscape.co +beauby.ru +becuo.com +bensbargains.net +best-mam.ru +bestdooz.com +bestpriceninja.com besttorrentknifta.weebly.com -bif-ru.info +bezcmexa.ru +big-cash.net bigames.online -bimatoprost-careprost.com -bimmerpost.com -binomo.com -blackbabesporn.com -bombla.org -bonzbuddy.com -brainboosting.club -brainxs.us -briomotor.co -bryansk.zrus.org -buckspar.gq -budilneg.xyz -burger-tycoon.com -butstrap.space +bio-optomarket.ru +bjanshee.ru +bkgr.se +blackhatworth.com +blog.f00kclan.de +bloggen.be +blogrankers.com +bnt-team.com +bochemit.com.ua +boeuklad.com +bolezniorganov.ru +boltalko.xyz +boobsimge.com +bradleylive.xyz +brianhenry.co +brus-vsem.ru +bumble.cheapwebsitehoster.com +bumskontakte.org +buron.pw +business-made-fun.com +buyclomidonlaine.com +buydissertation.net +buyk.host.sk buypuppies.ca -buzzurl.jp -bytimedance.ru -calc-for-credit.ru -candycrushshop.com -candypeople.se -cars-modification.net -carsplate.com -cartierlovestore.com -centrdebut.ru -cezartabac.ro -chastnoeporno.com -chatroulette.online -cheatcode-lita12.rhcloud.com -checktext.org -chelnytruck.ru -citysecurity.nu -clarithromycin500mg.com +bwlx.prepedu.cn +byme.se +calcularpagerank.com +calstaterealty.us +capture-room.com +cartierloveringreplica.xyz +cayado.snn.gr +cbb1smartlist12.click +cdnnetwok.xyz +cejewelry.xyz +censys.io +cerev.info +chatseo.com +chcu.net +chorus.terakeet.com +christianlouboutinreplica.pw +cialgenisrx.com +cialischmrx.com +cialisndbrx.com +cicaki.net +cigarpass.com classiquebijoux.ru -clevernt.com -clickbank.net -clickbanksites.info -clickhype.com -club-musics.ru -cmhomestayagency.com -companies-catalog.com -compliance-irvin.xyz -computernetworksonline.com -conciergegroup.org -converse.ddsoldes.fr -coolyarddecorations.com -counterbot.com -countercrazy.com -covadhosting.biz -credit-respect.ru -crirussian.ru -culturevie.info -dandiyabeats.in -db.speedup.gdn -decenttools.com +cleansearch.net +clickso.com +cobaltpro.ru +compliance-nelson.top +cookie-law-enforcement-hh.xyz +coolwallpapers-hd.com +copyrightclaims.org +crackzplanet.com +culad.com +cupday.com +customboxes4less.com +customsua.com.ua +cyprusbuyproperties.com +cyse.tk +daretodonate.co +datract.com +davidovic.info +dcj-nn.ru +delayreferat.ru +delitime.info +deluxedumps.com +demon-tweeks.com +derevesendeco.com +deshevo-nedorogo.ru design-lands.ru +detskie-zabavi.ru +detsky-nabytek.info +deutschland123.de dev-seo.blog -dfwu1019.info -diarioaconcagua.com -dieswaene.com -divci-hry.info -dmmspy.com -dnepropetrovsk.mistr-x.org -docs4all.com -doctissimo.fr -doctormakarova.ru -documentsite.net -dogoimage.com -domaineaneblanc.com -domainsatcost.com -dominterior.org -e-buyeasy.com -ebooknovel.club -edge.sharethis.com -educontest.net -el-nation.com -elektrischeziga.livejournal.com -elektrischezigarette2.onsugar.com -eloxal.ru -ereko.ru -erotiktreff24.info -exlarseva.webblog.es -exoneration-project.us -expolicenciaslatam.co -extlinks.com -ezofest.sk -f5mtrack.com -fba-mexico.com -fikasound.tk -findpik.com -findthe.pet -foodcrafts.website -fordsonmajbor.cf -forensicpsychiatry.ru -fortwosmartcar.pw -foto-telok.net -fotosfotos.eu -foxjuegos.com -frbizlist.com -frcls.fr -free-traffic.xyz -freeseedsonline.com -freshmac.space -front.to -fugarif.ga -fullfileaccess.com -fullgirl.ru -funnymama.com -gaypornmovie.net -gaz-voshod.ru -gdebestkupit.ru -genericviagrasildenafiled.net +diesel-parts28.ru +digitalassetmanagement.site +diy-handmade-ideas.com +drev.biz +eaplay.ru +educatemetv.com +elektrischezigarettekaufen2.cowblog.fr +elektrozigarette2.webs.com +elitedollars.com +englate.com +epngo.bz +es5.com +euromasterclass.ru +european-torches.ru +event-tracking.com +evrotekhservis.ru +extractorandburner.com +ezigarettenkaufen2.dreamwidth.org +ezigarettenshop2.postbit.com +face.hostingx.eu +faithe.top +falcon-images.blogspot.com +fetroshok.ru +filmetricsasia.com +filmidivx.com +firesub.pl +firstsiteguide.com +flex4launch.ru +floating-share-buttons.com +floridahuntingfishingadventures.com +flytourisme.org +forex.osobye.ru +fortevidyoze.net +forums.toucharcade.com +free-fb-traffic.com +freecamdollars.com +freewebs.com +froggytube.com +fsalas.com +galleryawesome.com +game-mmorpg.net +gazporno.com get-seo-domain.com -get-your-social-buttons.info -getmyads24.com -gettpromos.com -gheus.altervista.org -goldadpremium.com -goodhousekeeping.com -googlepositions.com -googleseo.com.tr -gotwebsite1.com -gq-catalog.gq -greatidea.marketing -greenzaim.ru -gribkovye-zabolevaniya.com -gta-club.ru -hamptonoaks.ca -hao123.com -hdimagegallery.net -healgastro.com -heliko.no -hermesbracelets.xyz -hetmanship.xyz -hollywoodactress.info -homeart.space -homik.pw -honyaku.yahoofs.jp -hothor.se -hotxnights.info -howmuchdoestizanidinecost.blogspot.com -huhn.altervista.org -hyipmanager.in -ibb.com.ua -idealtits.net -ideashome.id -imagui.eu -img.wallpaperstock.net -imgarit.pw -importchinacoach-teach.com -inclk.com -infodocsportal.com -intervsem.ru -invitefashion.com -isitwp.com -islandminingsupply.wordpress.com -ispaniya-costa-blanca.ru -istizanidineanarcoticdrug.blogspot.com -istock-mebel.ru -ivearchenceinflu.cf -iyasimasennka.com -javrip.net -javspace.net -jerseyschinabizwholesale.us -jestr.org -jjbabskoe.ru -jo24news.com -jose.mulinohouse.co -justprofit.xyz -jwcialislrt.com +getmiro.com +gg-arena.ru +gidonline.one +golden-catalog.pro +goldenggames.com +gombita.info +goodwinmetals.co +gopro-online.info +graphid.com +gsbs.com.ua +gsmtlf.ru +gta-top.ru +gwhwpxbw.bloger.index.hr +harmonyglen.us +hawaiielectriclight.com +highland-homes.com +hit-kino.ru +hoholikik.club +homemature.net +hotloans.ru +hotsocialz.com +hplaserjetpdriver8y.pen.io +hreade.com +hydropump.su +hyhj.info +idegenvezeto.eu +iklysha.ml +images-graphics-pics.com +imfamous.info +inboxdollars.com +interferencer.ru +investingclub.ru +investyb.com +invivo.hu +irkutsk.zrus.org +iscblog.info +ispac.org +itag.pw +itis4you.com +izismile.com +jackwolfskinoutlet.online +jav-idol.com +javcoast.com +johannesburgsingles.co.za +jongose.ninja jwss.cc -kashubadesign.ru +kaac.ru +kakadu-interior.com.ua +kamagragelusa.net +kamorel.com kazan.zrus.org -kerei.ru -ketrzyn.pl -keywordspay.com -kfon.eu -kino2018.club -klumba55.ru -komputernaya-pomosh-moscow.ru -kostenlos-sexvideos.com -koversite.info -krassh.ru -krynica.info -kursy-ege.ru -labvis.host.sk -lafriore.ru +klin3952.ru +klosetkitten.com +krafte.ru +lanasshop.ru +landinez.co +landoftracking.com larchik.net -letolove.ru -lincolntheatre.com +law-check-nine.xyz +lenvred.org linkrr.com -lisque.batcave.net -listiseltemournlan.gq -loftdigital.eu -lol-smurfs.com -losangeles-ads.com -lostaruban.com -lostfilm-online.ru -magnetic-bracelets.ru -mansparskats.com +live-cam6.info +lollypopgaming.com +lonely-mature.com +lottospring.com +love-baby.cz +lumb.co +lunamedia.co +lyrster.com +m0r0zk0-krava.ru +macresource.co.uk +manualterap.roleforum.ru marinetraffic.com -masturbate.co.uk -mature.free-websites.com -mcadamssupplyco.com -meetlocalchicks.com -mega-polis.biz.ua -megahdporno.net -merryhouse.co.uk -meteocast.net -mil-stak.com -monclercoats.xyz +master-muznachas.ru +mastershef.club +matb3aa.com +meble-bogart.info +medicinacom.ru +meds-online24.com +megakino.net +microstatic.pl +millionare.com +minecraft-neo.ru +minecraft-rus.org +minet.club +mirobuvi.com.ua +missclub.info +mitrasound.ru +mnogabukaff.net +mobile-appster.ru +mobile.ok.ru +mobioffertrck.com +modenamebel.ru +moesexy.com +mompussy.net monclerjacketsoutlet.pw -moncleronline.xyz -monetizer.com-01.site -mossmesi.com -motherboard.vice.com -movies-in-theaters.net -moviezbonkerssk.cf +morlat.altervista.org +morocco-nomad-excursions.com +moyakuhnia.ru +moyaterapiya.ru +mp3ringtone.info +mrlmedia.net mrwhite.biz -mug-na-chas-moscow.ru -muz-tracker.net -muztops.ru -mxgetcode.com -mycouponizemac.com -mygameplus.com -myseoconsultant.com -nardulan.com -naturtreenspicerx.pw -neks.info -nerudlogistik.ru -networkad.net -newstraveller.ru -newyorkhotelsmotels.info -nicefloor.co.uk -normalegal.ru -nyfinance.ml -nzfilecloud.weebly.com -o-dachnik.ru -o-o-8-o-o.com -obsessionphrases.com -ochistka-stokov.ru +mundoaberrante.com +mustat.com +my-cash-bot.co +mybinaryoptionsrobot.com +myfreecams.com +myfreetutorials.com +myghillie.info +myhydros.org +mylovelibrabry.com +myshopmatemac.com +namenectar.com +naphukete.ru +newsperuse.com +nicovideo.jp +nl.netlog.com +nrjmobile.fr +ntic.fr +nvformula.ru +o.light.d0t.ru +obesidadealgarve.com oda.as -ok-ua.info -one-gear.com -online-sbank.ru -online-templatestore.com -onlinedomains.ru -ooo-gotovie.ru -optom-deshevo.ru -oralsexfilme.net -ostroike.org -page2rss.com -panchro.co.uk -papasdelivery.ru -paperwritingservice17.blogspot.ru +odywpjtw.bloger.index.hr +okmusic.jp +onlyforemont.ru +origin-my.ru parajumpersoutlet.online -pattersonsweb.com -pc-services.ru -pcimforum.com -pdns.cz -perl.dp.ua -pestomou.info -petedrummond.com -phuketscreen.com -pictures-and-images.com -pinup-casino1.ru -piulatte.cz -pixshark.com -play-mp3.com +park.above.com +pay2me.pl +pensplan.com +photo.houseofgaga.ru +picscout.com +pictures-and-images.net +pinkduck.ga +pinwallpaper.xyz +piter.xrus.org +pizda.lol pochemychka.net -poffet.net -pokemongooo.ml -pomoc-drogowa.cba.pl +poisk-zakona.ru +popserve.adscpm.net +porndl.org pornhub-ru.com porno-play.net -porno-video-chati.ru -pornoforadult.com -postclass.com -princeadvantagesales.com -princevc.com -prizestohandle.club -prodvigator.ua -projectforte.ru -prostitutki-belgoroda.org -purplesphere.in -pussysaga.com -puteshestvennik.com -qualitymarketzone.com -ranking2017.ga -rapidgator-porn.ga -razvratnoe.org -realmonte.net -recinziireale.com -reeyanaturopathy.com +pornogad.com +pornohub.me +pornosee.info +pornosok.ru +pornotubexxx.name +pornstartits.xblog.in +privatbank46.ru +proekt-mos.ru +profitkode.com +pronekut.com +propranolol40mg.blogspot.com +psoriasis-file.trade +radiodigital.co +ranksonic.org +razorweb-a.akamaihd.net +real-time-analytics.com +realtytimes.com +refudiatethissarah.info +regionshop.biz registratciya-v-moskve.ru -reklama1.ru -rent2spb.ru -resant.ru +registrationdomainsite.com +registrydomainservices.com +reining.lovasszovetseg.hu +remont-comp-pomosh.ru +renecaovillasale.online +rentalcarnavi.info +repeatlogo.co.uk +responsive-test.net +restorator-msk.ru resultshub-a.akamaihd.net -rialp.getenjoyment.net -rique.host.sk -risparmiocasa.bz.it -rixpix.ru -rockingclicks.com -rockma.se -roll123.com -rubbed.us -rukino.org -rus-teh.narod.ru -s.lollypopgaming.com -sa-rewards.co.za -saddiechoua.com -saecsa.co -sanyuprojects.com -sbornik-zakonov.ru -scanner-alexa.top -scanner-olga.top -school-diplomat.ru -scripted.com -semalt.com +reversing.cc +rfd-split.hr +roleforum.ru +rucrypt.com +running-line.ru +russia-today-video.ru +russian-postindex.ru +rustic-quiver.win +ryetaw.com +s-iwantyou.com +sashagreyblog.ga +satellite.maps.ilovevitaly.com +scanner-elena.top +scenicmissouri.us +scrnet.biz.ua +sdelai-prosto.ru +se-welding.ru +search-error.com +searchkut.com +searchwebknow-a.akamaihd.net semp.net -semprofile.com -seoanalyses.com -seojokes.net -seorank.info -seozoom.it -sexflirtbook.com -sexpartygirls.net -shareyards.com +seocheckupx.com +seonetwizard.com +sexkontakte-seite.com +sexkvartal.com +sexsaoy.com +sexuria.net shariki-zuma-lines.ru -shoesonlinebuy.cn -shopvilleroyboch.com.ua -sibtest.ru -simple-image.com.ua -simple-share-buttons.com -simplesite.com -sisi-go.ru -site-speed-check.site -skanninge.se -sklad-24.ru -skladvaz.ru -skuteczna-dieta.co.pl -skutecznetabletkinaporostwlosow.pl -slim.sellany.ru -sloopyjoes.com -sluganarodu.ru -snow.nvr163.com -snowplanes.com -social-s-hhh.xyz +shop-electron.ru +shoppingmiracles.co.uk +siteonomy.com +sk.golden-praga.ru +smartadserver.com +smartshoppymac.com socialmadesimple.com -socialmediasuggest.com -softxaker.ru -songoo.wz.cz -sonnikforme.ru -spacash.com -spaceshipad.com -spravka130.ru -spy-app.info -srecorder.com -srgwebmail.nl -stard.shop -sterva.cc -strfls.com +sodexo.com +soft-program.com +sogimlecal.tk +solicita.info +solitaire-game.ru +sovetogorod.ru +space2019.top +sprttrack.com +spy-sts.com +spyfu.com +spywareit.com +stairliftsarea.com +starwars.wikia.com +stats-public.grammarly.io +stockquotes.wooeb.com +streamin.to studworks.org -styro.ru -surflinksmedical.com +surfbuyermac.com svolze.com -tech4master.com -techart24.com -teenfuck.tv -telegraf.by -telemetryverification.net -texbaza.by -tfxiq.com -thaisamkok.com -the-torrent-tracker.blogspot.com -theprofitsmaker.net -thetoiletpaper.com -tizanidine4mgprice.blogspot.com -tizanidine4mgstreetprice.blogspot.com -tizanidineforopiatewithdrawal.blogspot.com -tizanidinehcl4mgtabinfo.blogspot.com -tizanidinehydrochloride2mg.blogspot.com -tizanidinesideeffects.blogspot.com -tomatis.gospartner.com -top-l2.com -top10registrycleaners.com -topcasinoratings.ru -turizm.bz -tvand.ru -tversvet.ru -uchetunet.su -ugogo.info -ukrtextbook.com -umg-stroy.ru -unitygame3d.com -univerfiles.com +t-bygg.com +taaaak.com +teletype.in +templates.franklinfire.co +thelottosecrets.com +themestotal.com +thepokertimer.com +thesmartsearch.net +thetattoohut.com +timer4web.com +tizanidineingredients.blogspot.com +tizanidinevsvalium.blogspot.com +tmm-kurs.ru +totu.us +trafficjunky.com +trafficnetzwerk.de +tripper.de +trucri.me +truebeauty.cc +twu.com.ua +u-cheats.ru +u17795.netangels.ru +ukrtvir.com.ua +ukrup.com +ultimatesetnewfreeallsoftupgradesystems.pw +unionmarkt.de unrealcommander.org -upproar.com -upupa.net -vapomnoncri.tk -varikozdok.ru -vellings.info +upstore.me +urlcut.ru +urldelivery.com +usiad.net +ussearche.cf +vacances-voyages.info +veerotech.com +venerologiya.com +venta-prom.ru +via-energy-acquistare.com +viagra.pp.ua +viddyoze.com video-chat.in +video-production.com +videochat.mx +vigrx-original.ru +vikistars.com vip2ch.com -vipromoffers.com -virtuagirl.com -voditeltrezviy.ru -volgograd.xrus.org -vonradio.com -voprosotvet24.ru -vsexkontakte.net -vvon.co.uk -vykupavto-krasnodar.ru -watchdogs-2.ru -wdrake.com -webenlace.com.ar -webix.me -webtherapy.ru -weightatraining.com -welck.octopis.com -wieseversa.no -witherrom55.eklablog.fr +vkontaktemusic.ru +vse-pesni.com +vykup-avto-krasnodar.ru +vzlom-na-zakaz.com +w7s.ru +wahicbefa31.soup.io +website-stealer.nufaq.com +westermarkanjou.se +westum.se worldis.me -writingservices17.blogspot.ru -wstroika.ru -www.888.com -www.event-tracking.com -wyniki-lista.pl -x-diesel.org -xev.ru -xingzi-vision.com -xn-----7kcabaipgeakzcss7bjdqdwpfnhv.xn--p1ai +woweb.com.ua +wpsecurity.website +www.pinnacle-bets.com +xclicks.net +xjrul.com +xkaz.org xn-----7kceclhb4abre1b4a0ccl2fxch1a.xn--p1ai -xn----9sbebi2bvzr7h.xn--p1ai -xn--1-8sbcpb0bdm8k6a.xn--p1ai -xn--d1acah0c.xn--p1ai -xn--oogle-wmc.com -xrus.org -xxxguitars.com -yourothersite.com -yoursite.com -zajm-pod-zalog-nedvizhimosti.ru -zapatosenventa.info -zawyna.ua -zigarettenonlinekaufen2.drupalgardens.com -zirondelli.it -znakomstva-piter78.ru -znaturaloriginal.com -zone-kev717.info -zoogdiesney.com -zoolubimets.ru -zscalerone.net -zscalertwo.net -zvetki.ru +xn----7sbho2agebbhlivy.xn--p1ai +xn--80aaafbn2bc2ahdfrfkln6l.xn--p1ai +xn--80aaagvmjabrs1aoc9luc.xn--p1ai +xn--80ahvj9e.xn--p1ai +xn--80ak6aa92e.com +xn--90acenikpebbdd4f6d.xn--p1ai +xnxx699.com +xtraffic.plus +xtubeporno.net +yachts-cruise.info +yeartwit.com +yellowproxy.net +youradexchange.com +youradulthosting.com +youtubedownload.org +zajm-zalog-krasnodar.ru +zakonobosago.ru +zeroredirect12.com +zeroredirect2.com +zeroredirect7.com +zolotoy-lis.ru +zoogdisany.com +zrelaya.pw diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index 648e8e255..7e9ff8749 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -1-99seo.com -18ps.ru -3-letter-domains.net -4kplayer.pl -4webmasters.org -acheterviagrafr24.com -adamoads.com -adman.se -adserver-e7.com -adtech.fr -adultfullhd.com -adultnet.in +1001desert.com +13tabs.com +214jaluwobapef.cf +3rbseyes.com +83net.jp +8gold.com +adinterax.com +adsfresh.men +advertisingtag.net ageofclones.com -ahrntal.verymes.xyz -akita.kz -aosheng-tech.com +aludecor.info +animalrank.com +apisecretsaucebi-a.akamaihd.net appfixing.space -appleid-verification.com -arcteryxstore.online -artavenuegardenstudios.com -atovh.local-finders.com -audiofree.ru -ausmepa.org.au -auto-zapchasti.org -autogrep.ru -autoseo-traffic.com -avto-oligarh.ru -avtocredit-legko.ru -ayerbo.xhost.ro -b00kmarks.com -backgroundpictures.net -beachdriveblog.com -beremenyashka.com -besthomemadepornsites.com -bgviagrachrx.com -bitporno.sx -blackhatworth.com -bluesalt.co -bodybuilding-shop.biz -bosman.pluto.ro -buttons-for-your-website.com -buyhoverboard.com -buynorxx.com -calvet.altervista.org -capture-room.com -chastnoeporno.com -cialgenisrx.com -clevernt.com -compliance-george.top -compliance-nelson.top -conocer-sanabria.com -coolwallpapers4k.info -copenergo.ru -copro.pw -crazyprotocol.com -crd.clan.su -dairyindia.in -darodar.com -dekorkeramik.ru -delayreferat.ru -dellalimov.com -demenageur.com -downtuptv.gq -dtnlyss.com -easync.io -ecommerce-seo.org -ekspresihati.info -elektrischezigaretten2.beeplog.com -esfchat.tk +autotrends.today +azazaporn.com +azbukafree.com +baksman.com +bankofthewext.com +belinka.com.ua +bestcurs.org +bestsexyblog.com +bestssaker.com +blackwitchcraft.ru +blogrankers.com +bonzibuddi.com +brandedleadgeneration.com +budpost.com.ua +buehne-fuer-menschenrechte.de +bumskontakte.org +c1.onedmp.com +candelluxsklep.pl +carapics.com +cardul.ru +cartierjusteunclou.xyz +cartierloveringreplica.xyz +cascadelink.org +cb.iphantom.com +cercacamion.it +com-cleaner.systems +com-onlinesupport.site +compliance-elena.xyz +compliance-mark.top +contentlook.co +cookielawblog.wordpress.com +crackzplanet.com +cv.wallhade.co +dealitright.click +dentalpearls.com.au +detmebel.su +dgfitness.co +docsportal.net +doktoronline.no +downloaddy.net +drivotracker.com +droidlook.net +druzhbany.ru +druzhininevgeniy63.blogspot.com +e-commerce-seo1.com +ekaterinburg.xrus.org +eloxal.ru +erot.co +errorfixing.space +euroskat.ru +expertblog.info extlabs.io -ezigaretteshop2.vefblog.net -falllow.gq -farmprofi.net +ezaz.info +fanpagerobot.com +fashionavenuegame.com +ferrotodo.com +fetlifeblog.com +filesclub.net +filesvine.com filmgo.ru -find1friend.com -findsexguide.com -finemanteam.com -fishtauto.ru -forexgb.ru -fyxabomiw.ru -galeria-zdjec.com -game-top.su -gamingspark.com -gazobeton-p.com.ua -genericlowlatencyasiodriverhq.aircus.com -get-seo-domain.com -goodprotein.ru -goodwriterssales.com -gpirate.com -graphics8.info -greatdealshop.com -green-tea.tv -gsasearchengineranker.space +filmidivx.com +financeloan.us +firstdrugmall.ru +flavors.me +fortunejack.com +fotopop.club +free-today.com +front.ru +funtoonez.com +gamedayhouse.com +gate5.co.za +gfaq.ru +glopages.ru +googlsucks.com +greatfind-a.akamaihd.net +gsou.cf +gwhwpxbw.bloger.index.hr h2monline.com -hcate.com -hdapp1008-a.akamaihd.net -hifidesign.ru -histats.com -hollywoodactress.info -homedecorpicture.us -hoporno.com -houston-vikings.com -hyipmanager.in -import-sales.com -incolors.club -investmac.com -it-max.com.ua -ivoiretechnocom.ci -javcoast.com -johannesburgsingles.co.za -juliaworld.net -knogg.net -kol-energo.ru -kosova.de -krasivoe-hd.net -krasnodar-avtolombards.ru -kvartira-sutochno.com -kw21.org -landscapebackgrounds.blogspot.com -linkwithin.com +hacktougroup.ru +hahashka.ru +hasfun.com +healgastro.com +heliko.no +hoverboard360.de +ibb.com.ua +ilikevitaly.com +imagefinder.site +imagine-ex.co +imgata.com +imitex-plus.ru +incomekey.net +infobabki.ru +investsuccess.org +investyb.com +iradiology.ru +ispac.org +jabimgo.pw +james13prix.info +kazan.zrus.org +keywordglobal.co.za +kidskunst.info +kinomaniatv.pl +kiwi237au.tk +kletkimehan.ru +kolotiloff.ru +krafte.ru +ledpolice.ru littleberry.ru -littlecity.ch -live-sexcam.tk -liveinternet.ro -lkbennettoutlet.online -llastbuy.ru -locksmith.jp -lomza.info -lotto6888.com -magiadinamibia.blogspot.com -makis.nu -map028.com -marktforschung-stuttgart.com -mhtr.be -mini-modus.ru -mobot.site -modabutik.ru -mountainstream.ms -movies-in-theaters.net -muz-baza.net -muznachas-service.ru +lkbennettstore.online +lodki-pvh.dp.ua +lunamedia.co +m-google.xyz +mabdoola.blogspot.com +masturbate.co.uk +maximpartnerspr.com +mebelcomplekt.ru +miniads.ca +mister-shop.com +mmoguider.ru +mobilemedia.md +moinozhki.com +moneymaster.ru +moneyzzz.ru +montredemarque.nl +mowser.com mydirtystuff.com -mylesosibirsk.ru -mystats.xyz -nardulan.com -nfljerseyscheapchinabiz.com +mywallpaper.top +narkologiya-penza.ru +net-radar.com nordvpn.com -nufaq.com -obiavo.uz -of-ireland.info -offtime.ru -okonich.com.ua -ororodnik.goodbb.ru -osagonline.ru -p-business.ru -paleohub.info -paperwritingservice17.blogspot.ru -partner-cdn.men -partner-pop.men -pb-dv.ru -pelfind.me -picturesfrom.com +nosecret.com.ua +o-dachnik.ru +o-o-8-o-o.ru +obiavo.ru +ooo-gotovie.ru +paclitor.com +paradontozanet.ru +partybunny.ru +pekori.to +pix24x7.com pladform.ru -plastgranar.nu -polimga.pw -popander.mobi -pornonik.com -pornoslive.net +polybuild.ru +porno-transsexuals.ru powenlite24.ru -predmety.in.ua -privhosting.com -pro-poly.ru -producm.ru -produkto.net -prostitutki-tolyatti.org -ptr.ruvds.com -purchasepillsnorx.com -putevka24.ru -quick-offer.com +predominant-invent.tk +privatefx-in.ru +prizesbook.online +progonrumarket.ru +progress-upakovka.ru +prokotov.com +prostitutki-yaroslavlya.org +pst2017.onlinewebshop.net +puzzleweb.ru +quickbuck.com quickcashlimited.com -renecaovillasale.online -respublica-otel.ru -rightenergysolutions.com.au -rniaeba.ga -rossanasaavedra.net -scrnet.biz.ua -seoholding.com -sex-watch.com -sexgalleries.top -sexsaoy.com -sexyebonyteen.com -sharebutton.to -shillyourcoins.com -shoppingjequiti.com.br -shoppytoolmac.com -sispe.com.br -site-auditor.online -skincrate.net -skyway24.ru -sledstvie-veli.net -sobecjvuwa.com.ru -sockshare.net -spammen.de -standardchartered-forex.com -staticfs.host -stockquotes.wooeb.com -strfls.com -stuff-about-money.com -sundrugstore.com -surflinksmedical.com -taxi-v-eisk.ru -teenforporn.com +r-e-f-e-r-e-r.com +razyboard.com +rcb101.ru +realtytimes.com +redhotfreebies.co.uk +revistaindustria.com +ritlweb.com +rukino.org +sarf3omlat.com +sayyoethe.blogspot.co.za +sbtdesign.co.uk +scanner-andrew.top +scottbywater.com +scrapinghub.com +searchkut.com +seo-2-0.com +seocheckupx.com +seocheki.net +service-core.ru +sex-znakomstva.online +sexflirtbook.com +sexkontakteao.info +shinikiev.com.ua +siha.de +simpoed.ufop.br +sisi-go.ru +siteheart.net +sizeplus.work +soc-econom-problems.ru +soft-terminal.ru +spaceshipad.com +spb.afora.ru +spin2016.cf +sport7777.net +sprttrack.com +spywarebegone.com +streetfooduncovered.com +stroymonolit.su +styro.ru +sugarkun.com +supermesta.ru +tds-advert005.info +telegraf.by theguardlan.com -tizanidineingredients.blogspot.com -top10-online-games.com -trackmedia101.com -traffictrade.life -truck-land.life -tv-spoty.info -tyumen.xrus.org -ukrtvir.com.ua -unitexindia.com -ussearche.cf -velpanex.ru +theheroes.ru +thin.me.pn +tizanidine4mgstreetvalue.blogspot.com +tizanidinehcl2mgsideeffects.blogspot.com +tonivedu.it +topclickguru.com +toursmaps.com +track112.site +traffic2money.com +trafficfactory.biz +treasuretrack-a.akamaihd.net +ukrtextbook.com +unrealcommander.com +urengoy.pro +urlopener.com +usswrite.com verymes.xyz -via-energy-acquistare.com -villacoloniale.com -villakohlanta.nu -vintontech.info -virtuagirl.com -vltai.com -vvpg.ru -vzlomtw.com -warningwar.ru -web-betting.ru -webinstantservice.com -website-stealer.nufaq.com -wetgames.ru -whatsappdownload10.com -www.get-free-traffic-now.com -x5market.ru -xep.info -xkaz.org -xn--d1abj0abs9d.in.ua -xnxx-n.com -yogamatsexpert.com -zakonobosago.ru -zeroredirect5.com -znakom.sibtest.ru -zvooq.eu +veselokloun.ru +video-woman.com +vikistars.com +vzlomfb.com +web.cvut.cz +webmasters.stackexchange.com +weddingdresses.xyz +wieseversa.no +wildworld.site +wiosenny-bon-1500.pl +workle.website +wtsindia.in +www.wohnkabinen-shop.de +xgames-04.com +xn--l1aengat.xn--p1ai +yginekologa.com +youradexchange.com +zoodrawings.com +zx6.ru +zygophyceous.womanstars.site diff --git a/dev-tools/test_units/ratelimittest-beta.txt b/dev-tools/test_units/ratelimittest-beta.txt index b2122ba99..3656720f2 100644 --- a/dev-tools/test_units/ratelimittest-beta.txt +++ b/dev-tools/test_units/ratelimittest-beta.txt @@ -1,11 +1,21 @@ - + + + +Welcome to NginX + + +
+

Welcome to the NginX Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + 503 Service Temporarily Unavailable

503 Service Temporarily Unavailable

nginx/1.18.0 (Ubuntu)
-ml> + 503 Service Temporarily Unavailable

503 Service Temporarily Unavailable

diff --git a/dev-tools/test_units/ratelimittest-master.txt b/dev-tools/test_units/ratelimittest-master.txt index f0474a154..3656720f2 100644 --- a/dev-tools/test_units/ratelimittest-master.txt +++ b/dev-tools/test_units/ratelimittest-master.txt @@ -581,4 +581,14 @@

Welcome to the NginX Ultimate Bad Bot Blocker

This is merely used for testing !!!

+ + + +Welcome to NginX + + +
+

Welcome to the NginX Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ \ No newline at end of file diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index 8fa087902..0774f55a5 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.236 0; +10.1.0.131 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index 68f51893f..10503a2fb 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -19966,3 +19966,1251 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2467 +### Updated: Sun Jul 11 14:05:08 UTC 2021 +### Bad Bot Count: 615 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: OPPO A33 +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ + diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 68f51893f..10503a2fb 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -19966,3 +19966,1251 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2467 +### Updated: Sun Jul 11 14:05:08 UTC 2021 +### Bad Bot Count: 615 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: OPPO A33 +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ +