From 1d2d5801958a714ed3e628b59ef680e449b98cca Mon Sep 17 00:00:00 2001 From: mitchellkrogza Date: Tue, 13 Jul 2021 14:22:50 +0000 Subject: [PATCH] V4.2021.07.2469 [ci skip] --- AUTO-CONFIGURATION.md | 2 +- MANUAL-CONFIGURATION.md | 2 +- README.md | 2 +- _google_analytics_ghost_spam/README.md | 2 +- .../etc/nginx/conf.d/globalblacklist.conf | 4 +- conf.d/globalblacklist-testing-version.conf | 4 +- conf.d/globalblacklist.conf | 4 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- dev-tools/buildnumber | 2 +- dev-tools/globalblacklist-testing.template | 4 +- dev-tools/globalblacklist.template | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- dev-tools/test_units/blacklist-ips.conf | 2 +- .../random-bots-for-test-quick.list | 20 +- .../test_units/random-bots-for-test.list | 296 ++-- .../random-bots-for-whitelist-test.list | 292 ++-- .../random-referrers-for-test-quick.list | 20 +- .../test_units/random-referrers-for-test.list | 948 ++++++------- .../random-referrers-for-whitelist-test.list | 488 +++---- dev-tools/test_units/whitelist-ips.conf | 2 +- dev-tools/tmp/robots.txt | 1248 +++++++++++++++++ robots.txt/robots.txt | 1248 +++++++++++++++++ 45 files changed, 3580 insertions(+), 1084 deletions(-) diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index 3243c64db..e3ddad8f5 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2468 +#### Version: V4.2021.07.2469 #### Bad Referrers Blocked: [7067](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index cb6a9d891..2dbc6eede 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2468 +#### Version: V4.2021.07.2469 #### Bad Referrers Blocked: [7067](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/README.md b/README.md index e59bb90d7..1901c685f 100755 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2021.07.2468 +#### Version: V4.2021.07.2469 #### Bad Referrers Blocked: [7067](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 161955f12..400d9c463 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,7 +3,7 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2021.07.2468 +#### Version: V4.2021.07.2469 #### Bad Referrers Blocked: [7067](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 6df822f92..7cb6c5a23 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2468 -### Updated: Mon Jul 12 14:05:19 UTC 2021 +### Version: V4.2021.07.2469 +### Updated: Tue Jul 13 14:17:00 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index b4d57644d..2d0a9829f 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2468 -### Updated: Mon Jul 12 14:05:19 UTC 2021 +### Version: V3.2021.07.2469 +### Updated: Tue Jul 13 14:17:01 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 6df822f92..7cb6c5a23 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2468 -### Updated: Mon Jul 12 14:05:19 UTC 2021 +### Version: V4.2021.07.2469 +### Updated: Tue Jul 13 14:17:00 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf index b4d57644d..2d0a9829f 100644 --- a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2468 -### Updated: Mon Jul 12 14:05:19 UTC 2021 +### Version: V3.2021.07.2469 +### Updated: Tue Jul 13 14:17:01 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 0d59e8a61..712736d7a 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.90 1; +10.1.0.168 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 981829a62..86cdae11d 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.90 0; +10.1.0.168 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index b4d57644d..2d0a9829f 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2468 -### Updated: Mon Jul 12 14:05:19 UTC 2021 +### Version: V3.2021.07.2469 +### Updated: Tue Jul 13 14:17:01 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 0d59e8a61..712736d7a 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.90 1; +10.1.0.168 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 981829a62..86cdae11d 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.90 0; +10.1.0.168 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index b4d57644d..2d0a9829f 100644 --- a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2468 -### Updated: Mon Jul 12 14:05:19 UTC 2021 +### Version: V3.2021.07.2469 +### Updated: Tue Jul 13 14:17:01 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 0d59e8a61..712736d7a 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.90 1; +10.1.0.168 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 981829a62..86cdae11d 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.90 0; +10.1.0.168 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index b4d57644d..2d0a9829f 100644 --- a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2468 -### Updated: Mon Jul 12 14:05:19 UTC 2021 +### Version: V3.2021.07.2469 +### Updated: Tue Jul 13 14:17:01 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index 1d2ff352b..636cf9ce0 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -2468 +2469 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index b4d57644d..2d0a9829f 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2468 -### Updated: Mon Jul 12 14:05:19 UTC 2021 +### Version: V3.2021.07.2469 +### Updated: Tue Jul 13 14:17:01 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index 6df822f92..7cb6c5a23 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2468 -### Updated: Mon Jul 12 14:05:19 UTC 2021 +### Version: V4.2021.07.2469 +### Updated: Tue Jul 13 14:17:00 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index c6becb50d..e5392509d 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -20.94.225.77 0; +20.191.103.52 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 6df822f92..7cb6c5a23 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2468 -### Updated: Mon Jul 12 14:05:19 UTC 2021 +### Version: V4.2021.07.2469 +### Updated: Tue Jul 13 14:17:00 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 0d59e8a61..712736d7a 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.90 1; +10.1.0.168 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 981829a62..86cdae11d 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.90 0; +10.1.0.168 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 6df822f92..7cb6c5a23 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2468 -### Updated: Mon Jul 12 14:05:19 UTC 2021 +### Version: V4.2021.07.2469 +### Updated: Tue Jul 13 14:17:00 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 0d59e8a61..712736d7a 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.90 1; +10.1.0.168 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 981829a62..86cdae11d 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.90 0; +10.1.0.168 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 6df822f92..7cb6c5a23 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2468 -### Updated: Mon Jul 12 14:05:19 UTC 2021 +### Version: V4.2021.07.2469 +### Updated: Tue Jul 13 14:17:00 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 0d59e8a61..712736d7a 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.90 1; +10.1.0.168 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 981829a62..86cdae11d 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.90 0; +10.1.0.168 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 6df822f92..7cb6c5a23 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2468 -### Updated: Mon Jul 12 14:05:19 UTC 2021 +### Version: V4.2021.07.2469 +### Updated: Tue Jul 13 14:17:00 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index c6becb50d..e5392509d 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -20.94.225.77 0; +20.191.103.52 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 06a423116..fe805a0e2 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2468 -### Updated: Mon Jul 12 14:05:19 UTC 2021 +### Version: V4.2021.07.2469 +### Updated: Tue Jul 13 14:17:00 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index c6becb50d..e5392509d 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -20.94.225.77 0; +20.191.103.52 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index d724d1520..62967e01a 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2468 -### Updated: Mon Jul 12 14:05:19 UTC 2021 +### Version: V4.2021.07.2469 +### Updated: Tue Jul 13 14:17:00 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index 0d59e8a61..712736d7a 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.90 1; +10.1.0.168 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 05e37e599..337509d58 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Anarchie -Drip -Ecxi -FrontPage -InterGET -Meanpathbot -Sosospider -WebStripper -ZmEu -crawler.feedback +BuzzSumo +DigitalPebble +GermCrawler +Nikto +ScreenerBot +ScrepyBot +Siphon +gopher +meanpathbot +seostar diff --git a/dev-tools/test_units/random-bots-for-test.list b/dev-tools/test_units/random-bots-for-test.list index dd77bd220..9ff4b88d0 100644 --- a/dev-tools/test_units/random-bots-for-test.list +++ b/dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -404enemy -80legs -ADmantX +404checker +ASPSeek Abonti -Aboundex -AhrefsBot Aipbot +Alexibot +Alligator +AlphaBot Anarchie -Anarchy -Ankit -Asterias +Anarchy99 +Aspiegel Attach +AwarioRssBot +AwarioSmartBot BDCbot BLEXBot +BackDoorBot BackWeb -BacklinkCrawler +Backlink-Ceck +Bandit Barkrowler BatchFTP +Battleztar Bazinga +BlackWidow Blackboard -BlowFish +Blow Boardreader -Brandprotect -Buck +Bolt +BotALot +Brandwatch BuiltBotTough BuiltWith -Bullseye BunnySlippers -CCBot +BuzzSumo +CATExplorador CSHttp -Calculon -CazoodleBot +Cegbfeieh CensysInspect -CheTeam -CheeseBot +ChinaClaw +Chlooe Cliqzbot +Cloud mapping +Cogentbot Collector +Copier CopyRightCheck Copyscape -CrazyWebCrawler +Cosmos +Curious Custo CyotekWebCopy -DTS Agent -Demon +DSearch +DataCha0s +Deusu +Devil DigitalPebble Dirbuster -Disco -DnyzBot -DomainAppender -EasyDL -Evil -Exabot -ExtLinksBot -FDM +Discoverybot +Dispatch +DomCopBot +DomainCrawler +DomainStatsBot +Domains Project +Download Wonder +Ebingbong +Ecxi +Express WebPictures FHscan -FemtosearchBot -Fimap -Flunky +Firefox/7.0 +FlashGet Freeuploader FrontPage Fuzz -GT::WWW -GalaxyBot -GetRight -GetWeb +Fyrebot +G-i-g-a-b-o-t +GermCrawler +Gigablast Gigabot Go!Zilla GoZilla -Gotit -GrabNet Grabber +Grafula GrapeFX -Haansoft -Harvest -Heritrix +HEADMasterSEO +HMView +HTTrack Hloader IDBTE4M -IDBot -IlseBot -IndeedBot -Indy Library +IRLbot +Id-search +Image Fetch +InfoTekies Intelliseek -InterGET +Internet Ninja InternetSeer -Iria IstellaBot -JetCar +JennyBot +JikeSpider Joomla -Jorgee -Jyxobot -Kenjin Spider Kinza Lanshanbot -Larbin -LeechFTP -Lftp -LibWeb -LieBaoFast -Likse +LeechGet +Libwhisker +LinkWalker Linkbot -LinkpadBot -Lipperhey -Litemage_walker -Lmspider -MIDown tool -MIIxpc +LinkextractorPro +LinqiaRSSBot +Ltx71 MJ12bot MQQBrowser -MSFrontPage -MSIECrawler MTRobot -Mag-Net Mail.RU_Bot -Majestic SEO -Majestic-SEO -Majestic12 -Mediatoolkitbot +MarkWatch +Mass Downloader +Mb2345Browser +Meanpathbot Metauri Microsoft Data Access -Microsoft URL Control Mister PiX +Moblie Safari +Mojolicious Morfeus Fucking Scanner +Mozlila +Mr.4x3 +Msrabot NICErsPRO NPbot Name Intelligence +Navroad NearSite -NetSpider -NetZIP -Netvibes -Nibbler +Nessus +NetLyzer +NetMechanic +Nettrack +NextGenSearchBot +Niki-bot Nuclei -Offline Explorer +Nutch +OPPO A33 OpenLinkProfiler Openfind -OutfoxBot -PHPCrawl -PageScorer -PageThing.com -Panscient -Picscout -Picsearch -PictureFinder +Page Analyzer +PageGrabber Pixray PleaseCrawl Pockey ProPowerBot -RankActive -RankActiveLinkBot -RankFlex -RankingBot +ProWebWalker +Psbot +Quick-Crawler +RSSingBot Re-re ReGet -Recorder -RepoMonkey -RocketCrawler -Rogerbot +SBIder +SEOkicks SEOkicks-Robot +SEOlyticsCrawler +SEOprofiler SEOstats +SISTRIX SMTBot -ScanAlert Scrapy -Screaming ScreenerBot ScrepyBot Semrush -Site Sucker -SiteCheckerBotCrawler -SiteExplorer +Seomoz +Siphon +SiteLockSpider SiteSnagger -SiteSucker Sitebeam Siteimprove +Sitevigil SlySearch -Sogou web spider +Snapbot +SocialRankIOBot +Sociscraper +SpaceBison Spammen -Spbot +SpankBot +Spanner Spinn3r SputnikBot -Sqworm -Steeler -SuperBot +Sqlmap +Sqlworm +Sucuri SuperHTTP -Suzuran -T8Abot -Telesoft +SurveyBot +Szukacz +T0PHackTeam +Teleport +TeleportPro +Telesphoreo +Telesphorep TheNomad +Thumbor TightTwatBot -Toweyabot Trendiction -Turingos -Turnitin -TwengaBot +TurnitinBot Twice -URLy Warning UnisterBot +Upflow V-BOT VB Project -VelenPublicWebCrawler -VoidEYE -Voil Voltron +WBSearchBot WISENutbot WWW-Collector-E +WWW::Mechanize WallpapersHD -WeSEE -Web Collage -Web Enhancer -Web Sauger -Web Sucker -WebAuto -WebCollage -WebFetch -WebFuck -WebGo IS -WebSucker -WebWhacker +Web Fuck +Web Pix +WebBandit +WebImageCollector Webalta +WebmasterWorldForumBot +Website Quester +WebsiteQuester +Webster Whack -Widow +Who.is Bot WinHTTrack -WiseGuys Robot Wotbox -Wprecon -Zauba -ZoomBot -ZumBot +YoudaoBot +ZoominfoBot ZyBorg arquivo-web-crawler +arquivo.pt +autoemailspider backlink-check -crawl.sogou.com -crawler.feedback -crawler4j -domainsproject.org +coccocbot-web +cognitiveseo +com.plumanalytics eCatch -evc-batch -facebookscraper -lwp-request +lwp-trivial magpie-crawler meanpathbot -mediawords -muhstik-scan -oBot +page scorer +pcBrowser +probe-image-size +ripz s1z.ru scalaj-http +seobility seoscanners -sexsearcher -siteripz -sp_auditbot -tAkeOut -trendiction.com -xpymep1.exe +seostar +serpstatbot +sitechecker.pro +sogouspider +trendiction.de +voyagerx.com +zauba.io diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 40e834305..05f1a1765 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -404enemy -80legs -ADmantX -Aboundexbot +360Spider +404checker +AIBOT +ASPSeek Acunetix -AhrefsBot +Aipbot AllSubmitter +Anarchie Anarchy -Ankit +Anthill Apexoo Aspiegel +Asterias +Attach +AwarioRssBot BDFetch -BLEXBot BackDoorBot -BackStreet -Bandit -Battleztar Bazinga +BackWeb +Backlink-Ceck +BacklinkCrawler +Badass +Barkrowler +BetaBot Bigfoot -BlackWidow -Blow -Brandwatch -BuiltBotTough +Blackboard +Buck +Buddy BuiltWith -BuzzSumo -CazoodleBot -CensysInspect +BunnySlippers +CODE87 +CSHttp +Calculon CheTeam +ChinaClaw +Chlooe +Cliqzbot Cloud mapping -Cocolyzebot -Cogentbot -Collector -CopyRightCheck -Cosmos -CrazyWebCrawler -Crescent +CrunchBot Custo -CyotekWebCopy -DIIbot -Demon +DTS Agent +DataCha0s Devil Digincore DigitalPebble -Disco DittoSpyder -DnyzBot -DomCopBot -DomainCrawler DomainStatsBot -Domains Project +Dragonfly +EMail Wolf EasyDL Ebingbong -Ecxi -EirGrabber EroCrawler -ExtLinksBot ExtractorPro Ezooms -Fimap -Flunky +FHscan +FlashGet +Freeuploader +FrontPage Fyrebot +GT::WWW GalaxyBot +Genieo GermCrawler -GetRight -Gigablast +Gigabot Go-Ahead-Got-It -GoZilla -Gotit -GrabNet -Grabber -Grafula -GrapeshotCrawler GridBot -HMView -HTMLparser -HTTrack -Harvest +HaosouSpider Humanlinks +HybridBot IDBot Iblog Image Fetch +Image Sucker IndeedBot Indy Library -InfoNaviRobot InfoTekies Intelliseek -InterGET -Iria -JOC Web Spider -JamesBOT -Jbrofuzz +Internet Ninja +InternetSeer +Iskanie +IstellaBot +JennyBot JetCar -JustView +Kenjin Spider +Kinza +LNSpiderguy Lanshanbot +Larbin +LeechFTP LexiBot -LibWeb -LieBaoFast -LinkextractorPro +Libwhisker +Likse +LinkScan +Linkdexbot +Lipperhey +Lipperhey Spider MFC_Tear_Sample -MIIxpc +MJ12bot +MQQBrowser +MSIECrawler Mag-Net -Majestic SEO -Majestic-SEO MarkMonitor +MarkWatch Mass Downloader -Mata Hari +Masscan +MauiBot +Mb2345Browser MeanPath Bot +Mediatoolkitbot MegaIndex.ru -MicroMessenger -Moblie Safari -Mojeek -Morfeus Fucking Scanner +Microsoft Data Access +Microsoft URL Control +Mojolicious Mozlila +Musobot +NICErsPRO +NPbot Name Intelligence -Nameprotect Navroad -Nessus -Netcraft -Nettrack +NearSite +Needle +Net Vampire +Netvibes +NextGenSearchBot +Nibbler +Nikto +Ninja Nmap Nuclei +OPPO A33 Offline Explorer -OrangeBot -PHPCrawl -POE-Component-Client-HTTP -Page Analyzer +OnCrawl +OpenLinkProfiler PageGrabber +PageScorer +Panscient +Papa Foto +PeoplePal Pi-Monster Picsearch -Piepmatz Pimonster +Pixray Pockey ProWebWalker -Probethenet -Pump -PxBroker PyCurl QueryN Metasearch +Quick-Crawler +RankActiveLinkBot RankFlex RankingBot2 RankurBot Re-re -ReGet -RebelMouse -RedesScrapy +RealDownload +Ripper SEOkicks -SEOkicks-Robot SEOlyticsCrawler SEOprofiler -SISTRIX -Scanbot -ScreenerBot +SEOstats +SalesIntelligent +ScanAlert +Scrapy +Screaming Searchestate SearchmetricsBot +Semrush SemrushBot +SentiBot +SeoSiteCheckup Seomoz -Shodan -Siphon Site Sucker -SiteCheckerBotCrawler -SiteSnagger +SiteLockSpider Sitebeam -SmartDownload -Snake -Snapbot -Sociscraper -SpankBot +SlySearch +Sottopop Spanner Spbot -Sqlworm -Sucker +Sqworm +Steeler Sucuri -SuperHTTP -Surfbot +SuperBot SurveyBot -Szukacz +T0PHackTeam Teleport -Telesphorep +TeleportPro +Telesphoreo The Intraformant -TheNomad -TightTwatBot -Toata -Trendiction -Trendictionbot -True_Robot -Turingos +Toweyabot +Tracemyfile Turnitin TurnitinBot -Twice +TwengaBot Typhoeus -URLy Warning URLy.Warning Upflow -VB Project +V-BOT +VCI Vacuum -Vagabondo -VidibleScraper -Virusdie -Voil -Voltron +VeriCiteCrawler +VoidEYE +WASALive-Bot WEBDAV -WPScan -Wallpapers/3.0 WallpapersHD -Web Collage +Web Auto +Web Enhancer Web Fetch Web Fuck -WebBandit -WebCopier -WebFetch +WebEnhancer WebFuck WebGo IS +WebImageCollector +WebLeacher WebPix -WebSauger +WebStripper WebWhacker WebZIP +Webalta +WebmasterWorldForumBot +Webshag +WebsiteExtractor WebsiteQuester -Whack -Who.is Bot -WiseGuys Robot +Webster +Whatweb +Widow +WinHTTrack Wotbox -Xaldon WebSpider -Xenu -Zade Zauba Zermelo -Zitebot +ZmEu ZoominfoBot -archive.org_bot -arquivo-web-crawler -backlink-check -check1.exe coccocbot-web -com.plumanalytics +crawl.sogou.com +crawler.feedback +crawler4j +demandbase-bot domainsproject.org -eCatch -evc-batch -instabid ips-agent -lwp-request -netEstate NE Crawler +lwp-trivial +mediawords +muhstik-scan +oBot pcBrowser -plumanalytics polaris version satoristudio.net -scan.lol seobility -sexsearcher -tAkeOut -trendiction.de -xpymep1.exe +sitechecker.pro +siteripz +spyfu +sysscan +trendiction.com +webmeup-crawler +x09Mozilla diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index c92a42213..0e618aefd 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -adtech.us -backlinkwatch.com -conciergegroup.org -costablue.xyz -galleryawesome.com -seoboxes.com -skatestick.bid -stmassage.ru -tizanidinehcl4mgtab.blogspot.com -witherrom55.eklablog.fr +bayareaaeroclub.org +countercrazy.com +decorazilla.com +mielec.pl +pivka.xyz +plastgranar.nu +sammyweaver.com +tizanidinestreetvalue.blogspot.com +treasuretrack-a.akamaihd.net +videotuber.ru diff --git a/dev-tools/test_units/random-referrers-for-test.list b/dev-tools/test_units/random-referrers-for-test.list index 6190b0ddd..26668e0b6 100644 --- a/dev-tools/test_units/random-referrers-for-test.list +++ b/dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -00author.com -123locker.com -123movies.love -1bet.com -1flag.co.za -1webmaster.ml -1xbet4you.com -2000k.ru -2020iscoming.info +03p.info +08800.top +1001desert.com +101raccoon.ru 23kw.ru -4istoshop.com +2drittel.de +4pp13.com +51.la 51unlim.ru -5u.com -70casino.online +7ooo.ru +7zap.com +8558l.jobs.net aa625d84f1587749c1ab011d6f269f7d64.com -above.com -adultnet.in -aero2.ru -agapovdg.ru -agardomains.com -akuhni.by -albumsuper.info -alevinefesleri.com -alfabot.xyz -algerianembassy.co.in -all-number.com -allnews.md -amazingpic.net -amung.us -an-donut.com -anal-acrobats.com -apartment.ru -apessay.com -appearance-cool.com -arcadepages.com -arenanews.com.ua -arendatora.ru -arvut.org -ascat.porn -asophoto.com -asrv-a.akamaihd.net -atovh.local-finders.com -autoblogger24.info -autotop.com.ua -autowebmarket.com.ua -aversis.net -aviav.co -babyfactory.fr -back.dog -backgroundpictures.net -backlink4u.net -badavit.com.ua -baladur.ru -balkanfarma.ru -baltgem.com -bayareaaeroclub.org -belsetirehafi.tk -bestofferswalkmydogouteveryday.gq -bestplacetobuyeriacta.jw.lt -betterhdporn.com -bezlimitko.xyz -bget.ru -big-cash.net -biketank.ga -bizlist.com.de -bizzliving.com -bjorkbacken.nu -blackle.com -bobinoz.com -bolezniorganov.ru -bouda.kvalitne.cz -bristolhotel.com -bronzeaid-a.akamaihd.net -burger-imperia.com -burger-tycoon.com -buyantiviralwp.com -buyviagraa.blogspot.com -bzero1jewelry.net -c1.onedmp.com -callawaygolfstore.online -carabela.com.do -cardsharp1.ru -cartierbracelet.xyz -cartierlovebraceletreplica.xyz -cashkitten-a.akamaihd.net +aa8b68101d388c446389283820863176e7.com +actionnooz.com +acunetix-referrer.com +addray.pro +adnotbad.com +adtech.de +adultgalls.com +advertisingtag.net +advokat-grodno.by +agysya.ru +ahrntal.verymes.xyz +airmaxshoes-2016.com +aksonural.ru +akusherok.ru +alldubai.biz +am-se.com +amazingninja.com +amigobulls.com +anaseracresar.tk +animal-drawings.com +apibetweenlinesn-a.akamaihd.net +apiholdingmypage-a.akamaihd.net +app5.letmacworkfaster.world +appfixing.space +apxeo.info +as5000.com +atlant-auto.info +autoplate.info +avkzarabotok.info +azbukafree.com +bablonow.ru +bagcionderlab.com +bangla-chat-uk-paralud.ga +bankiem.pl +belinka.com.ua +bezcmexa.ru +bezsporno.ru +bjgugu.net.ua +blackplanet.com +bmw-ark.ru +boc.kz +bolegapakistan.com +boleznikogi.com +boltalko.xyz +bonzibuddi.com +boostmyppc.com +brainzpod.win +breget74.ru +brianhenry.co +browserprotecter.com +burkesales.com +bus-offer.com +buyparajumpers.online +buyscabiescream.com +buzz-porno.info +bwlx.prepedu.cn +c-gainsbourg.com +cablecar.us +call-of-duty.info +candy-glam-hp.com +canrioloadm.gq cctva.tv -cheapcigarettesc.info -cheapjerseysap.com -cheapwebsitehoster.com -chocolateslim-u-srbiji.com -cialischsrx.com -cialischstgerts.com -clickzzs.nl -clodo.ru -compiko.info -compliance-don.top -compliance-don.xyz -comsysnet.com -connectionstrenth.com -coolwallpapers-hd.com -coslab.club -counterbot.com -customchocolate.business-for-home.com -dandiyabeats.in -darkbooks.org -day-news.info +cdnnetwok.xyz +cementaresearch.se +centraletermice.us +cheapjerseysbizwholesale.us +cheatcode-lita12.rhcloud.com +checkhit.com +chocoslim.pro +cialisndbrx.com +cinemaenergy-hd.ru +cleanersoft.com +cleanmypc.com +club-lukojl.ru +cocyq.inwtrade.com +comeondog.info +compliance-brian.xyz +compliance-checker.info +compliance-ivan.top +contextualyield.com +cookielawblog.wordpress.com +copyrightclaims.org +cornomase.win +counter.yadro.ru +cowblog.fr +cpabegins.ru +cracksplanet.com +crd.clan.su +creativehutindia.com +csgo4.win +culmimg.pw +customcatchcan.com +customergrowthsystems.com +cwetochki.ru +cyclobenzaprinevstizanidine.blogspot.com +dairyindia.in +danceuniverse.ru +date-withme.com +dating-app.ru dealwifi.com -dekoration.us -delitime.info -demenageur.com -demon-tweeks.com -derevesendeco.com -design-ideas.info -deutschehobbyhuren.net +delta-line.men +demian.kz +designdevise.com +detmebel.su +diffbot.com +dignitasdata.se +dipstar.org +directivepub.com discountbarbour.online -donna7753191.ru -donvito.unas.cz -downloads-whatsapp.com -driving.kiev.ua -dscaas.website -dverimegapolis.ru -e-kwiaciarz.pl -econom.co -ege-essay.ru -elektrischezigarette2.devhub.com -elektrischezigarettekaufen2.cowblog.fr -elektrozigarette2.wordpressy.pl -elexies.info -elitedollars.com -enhand.se -euromasterclass.ru -european-torches.ru -euroskat.ru -evrotekhservis.ru -excaliburfilms.com -exoclick.com -extlinks.com -extremez.net -ezigaretteshop.webs.com -fakehandbags.xyz -farm26.ru -fashionindeed.ml -fastfixing.tech -femdom.twiclub.in -fertilitetsradgivningen.se -fetishinside.com -fifa-coins.online -filkhbr.com -filmetricsasia.com -financepoints.eu -findanysex.com -fireads.men -flashlarevista.com -floridamobilebillboards.com -footbalive.org -fordsonmajbor.cf -fortwosmartcar.pw -foto-sisek.porngalleries.top -fotoxxxru.com -fr.netlog.com -franecki.net -frankofficial.ru -freegamesplay.online -freeuploader.ml -frequiry.com -fres-news.com -freshmac.space -freshwallpapers.info -froggytube.com -fullfileaccess.com -gcup.ru -get-free-traffic-now.com -getmiro.com -ghernnqr.skyrock.com -golmau.host.sk -goo.ne.jp -gosmeb.ru -gratuitxblcodes.com -greatgrace.ru -greatzip.com -grizzlysgrill.com -groupmoney.ru -gsasearchengineranker.xyz -gsasearchenginerankerdiscount.com -guildebzh.info -hdfreeporno.net -hdimagelib.com -hexpilot.com -host-tracker.com -hotelcrocenzi.sm -howopen.ru -humbmosquina.tk -ie.57883.net -ilbe.club -ilmexico.com -ilovevitaly.ru -imagerydatabase.com -impisr.edunsk.ru -impresagaia.it -inboxdollars.com +djonwatch.ru +domainanalyzing.xyz +domaincheck.io +domashneeruporno.com +dorratex.tn +dprtb.com +dptaughtme.com +easytuningshop.ru +easyukraine.com +editmedios.com +educatemetv.com +edudocs.net +eduinfosite.com +egovaleo.it +ekosmetyki.net.pl +eldiariodeguadalajara.com +elektronischezigarette2.mex.tl +elektrozigarettekaufen1.devhub.com +elkacentr.ru +elvenmachine.com +enpolis.ru +enternet.ee +eropornosex.ru +etm-consult.de +event-tracking.com +exct.net +extremal-blog.com +eyemagination.com +ezigarettekaufen2.yolasite.com +facecup.top +faithe.top +fanoboi.com +fastcrawl.com +firstsiteguide.com +fitfloponline.store +fiuxy.com +flash4fun.com +forexgb.ru +forexunion.net +fortunejack.com +forums.d2jsp.org +foxjuegos.com +franklinfire.co +freefoto.ca +frustrated-favorable.gq +funcrushgames.com +fyxabomiw.ru +gambarxkata.co +gay.adultgalls.com +gembird.com +getyourimage.club +girlporn.ru +given2.com +glall.ru +googleseo.com.tr +gsasearchengineranker.top +gymi.name +hackers-crackers.tk +handicapbathtubarea.com +hao123.com +hatedriveapart.com +heliko.no +herokuapp.com +historichometeam.com +hiwibyh.bugs3.com +hkdiiohi.skyrock.com +hobbyhuren24.net +hop.clickbank.net +host-protection.com +ictizanidinehcl4mg.blogspot.com +idegenvezeto.eu +ideoworld.org +igtools.club +ikearugs.xyz +images.gyffu.com +imicrovision.com +incanto.in.ua infogame.name -infospot.pt -instasexyblog.com -intellekt21.ru -investpamm.ru -ip-guide.com -iron-age.info -isistaylorporn.info -islandminingsupply.wordpress.com +inome.com.ua +interesnie-faktu.ru +inzn.ru +iplusbit.blogspot.co.za +iskalko.ru ispac.org -iwantedmoney.com -iweblist.info -izismile.com -jav-fetish.site -jav-way.site +itsdp3.com jbl-charge.info -jo24news.com -jobmarket.com.ua -jrcigars.com -jrpmakati.com -karadene.com -karting196.ru -kerwinandcariza.com -kihi.gdn -kinoflux.net -kiwi237au.tk -kladrus.ru -kleine-titten.biz -knowyournextmove.com -konkursov.net -krasivye-devushki.net -lacasamorett.com -lafriore.ru +joessmogtestonly.com +jofucipiku.tk +jus0wil.pp.ua +kalb.ru +kamorel.com +katadhin.co +kazan.xrus.org +kerei.ru +keywords-monitoring-your-success.com +kiinomaniak.pl +kimcurlrvsms.com +kinohall.ru +kinomaniatv.pl +koral.se +kozhniebolezni.com +krasnodar.zrus.org +kupiproday.com.ua +lacave.ntic.fr +ladov.ru lakomka.com.ua -lankarns.com -legionalpha.com -letto.by -lider-zhaluzi.kiev.ua -likesdesign.com -lineshops.biz -linkpulse.com -livingroomdecor.info -long-beach-air-conditioning.com -lost-alpha.ru -louboutinshoes.xyz -luckyshop.net.ua -mafa-free.com -magento-crew.net -makedo.ru -maladot.com -manve.info -markjaybeefractal.com -marmitaco.cat -mcnamaratech.com -mediafresh.online -medkletki.ru -meet-flirt-dating.com -megagrabber.ru -merryhouse.co.uk -mielec.pl -miniature.io -missvietnam.org -mobile.ok.ru -mojpreskumanie.com -monclercoats.xyz -monetizer.com-01.site -moneyteam24.com -moscow-region.ru -moviezbonkerssk.cf -moyakuhnia.ru -moz.com -mrbitsandbytes.com -musicktab.com +laptop-4-less.com +lasvegaslockandsafe.com +laulini.soclog.se +law-check-two.xyz +lechenie-gemorroya.com +lerporn.info +lezbiyanki.net +lifeinsurancekb.com +light.ifmo.ru +limtu.ifmo.ru +lincolntheatre.com +lineavabit.it +linkbolic.com +linkbuddies.com +linkbux.ru +lion.misba.us +liquimondo.com +live-cam6.info +livesex-amateure.info +lob.com.ru +localmatchbook.com +locationdesaison.com +lollypopgaming.com +lomb.co +lombia.com +longlifelomilomi.info +louboutinbooties.xyz +lubetube.com +lyrics.home-task.com +mac-shield.com +mackeeper-center.club +magazintiande.ru +mahnwachen-helfen.info +matb3aa.com +maximpartnerspr.com +mebel-vstroika.ru +medicineseasybuy.com +medmajor.ru +medosmotr-ufa.ru +meetlocalchicks.com +methodsmarketing.com +micasainvest.com +microstatic.pl +mir-business-24.ru +mitrasound.ru +mmofreegames.online +mobplayer.net +moinozhki.com +moneymaster.ru +moomi-daeri.com +morefastermac.trade +mustwineblog.com +muznachas-service.ru +mwtpludn.review +mydoctorok.ru mymobilemoneypages.com -narkologiya-orel.ru -narkologiya-voronezh.ru -national-today-winning-winner.club -naval.jislaaik.com -news-speaker.com +nalogovyy-kodeks.ru +netanalytics.xyz +next-dentists.tk +nextbackgroundcheck.gq nhl09.ru -ningessaybe.me -noumeda.com -nowtorrents.com -obiavo.in -obiavo.su -obsessionphrases.com -okayimage.com -onemantrip.com -onko-24.com -online247.ml -onlinetvseries.me -openmultipleurl.com -outclicks.net -outshop.ru -ownshop.win -oynat.info +nikhilbahl.com +no-rx.info +novatech.vn +novosti-hi-tech.ru +nuker.com +obiavo.ru +okmusic.jp +okroshki.ru +onblastblog.online +onemactrckr.com +optom-deshevo.ru +orion-v.com +ossmalta.com oyster-green.com -paceform.com -partner-print.men +paclitor.com +panchro.co.uk +paradontozanet.ru +parajumpersjakkesalgnorge.info +partner-high.men partner-stop.men partnerworkroom.men -pekori.to -pictures-and-images.net -picturesify.com +pay2me.pl +pc-services.ru +pc-test.net +pc4download.co +pestomou.info +pflexads.com +photochki.com +photosaga.info +picmoonco.pw +picturesboss.com picturesmania.com -pinapchik.com -pinkduck.ga -piuminiita.com -pk-pomosch.ru -ploenjitmedia.azurewebsites.net -pobeiranie.pl -pod-muzyku.club -popunder.ru -porno-play.net -porno.simple-image.com.ua -pornohd1080.online -pornosok.ru -pornpost.in -pornstartits.xblog.in -portside.cc -portside.xyz -potolokelekor.ru -pourvous.info -predmety.in.ua -professionaldieselcare.com -profolan.pl -promgirldresses.xyz -proposal-engine.com -prostitutki-yaroslavlya.org -proxyradar.com -pulse33.ru -quit-smoking.ga -qwesa.ru -rarbg.to -razleton.com -rednise.com -remmling.de -repeatlogo.co.uk +piter.xrus.org +pkr1hand.com +plastgranar.nu +platezhka.net +playmsn.com +pliks.pl +pogosh.com +pons-presse.com +popander.mobi +porno-video-chati.ru +pornokorol.com +pornosmola.info +pornozhara.com +porodasobak.net +preparevideosafesystem4unow.space +privatefx.all4invest.info +prizesbook.online +pro-tec.kz +prodvigator.ua +prohoster.info +prostitutki-tolyatti.org +proxyelite.biz +quelle.ru +rabot.host.sk +rainbowice.ru +rangoman.date +rankingchart.de +ranksonic.org +rasteniya-vs-zombi.ru +rating-bestcasino.com +rebuildermedical.com +redhotfreebies.co.uk +referencemoi.com +registry-cleaner.net +remont-comp-pomosh.ru +remont-komputerov-notebook.ru +rentehno.ru replicaclub.ru -responsinator.com -retreatia.com -robertefuller.com -rospromtest.ru -rp9.ru -ruf777.com -rukino.org -sabizonline.com -saddiechoua.com -sady-urala.ru -saveriopiazza.it -sbwealthsolutions.ca -scanmarine.info -scanner-fred.top -scanner-jack.top -search.1and1.com -secretscook.ru -seeingmeerkat.com -semprofile.com -seo-2-0.com -seoheap.com -seorankinglinks.us -seotoolsagency.com -sergiorossistore.online -servicecenter.co.ua -serving.adbetclickin.pink -sexstream.pl -sexystrippe.info -sfj-ror.no -simple-image.com.ua -simul.co -sitiz.club +revistaindustria.com +rfd-split.hr +richinvestmonitor.com +rocis.site +rockma.se +rusenvironmental.net +rusoft-zone.ru +satellite.maps.ilovevitaly.com +sawin.beth.webd.pl +sbtdesign.co.uk +schoolfiles.net +scmor.ilxc.cc +search.pch.com +searchaddis.com +searchencrypt.com +see-your-website-here.com +seobility.net +seoexperimenty.ru +seolab.top +sex.hotblog.top +sexobzor.info +sexsearch.com +sexxdate.net +sexytrend.ru +shop.acim.org +shymkent.xkaz.org +sibdevice.ru +silverage.ru +silvercash.com +simon3.ru +sinestesia.host.sk +siteheart.net sizeplus.work -slayerlife.com -social-buttons.com -social-search.me -softlinesolutions.me -softwaretrend.net -spabali.org -special-porn.com -stairliftsarea.com -stardevine.com -startufa.ru -starwars.wikia.com -statustroll.com -stauga.altervista.org -steamoff.net -stefanbakosab.se -sticken.co -stockspmb.info -storehouse.ua -strana-krasoty.ru -studworks.org -subj.ukr-lit.com +slavkokacunko.de +slujbauborki.ru +slutloadlive.com +smart-scripts.com +smartpet.ru +snjack.info +snow.nvr163.com +snyatie-lomki-v-stacionare.ru +soaksoak.ru +sockshare.net +sps-shop.com +srecorder.com +stathat.com +stmassage.ru +stoki.ru +store-rx.com +sugarlyflex.pw +sumo.com sundrugstore.com -supermesta.ru -superstarfloraluk.com -surgut.zrus.org -svtrd.com -taihouse.ru -taraz.xkaz.org -tedy.su -teenforporn.com -tehngr.ru -telefonsexi.com -telegramdownload10.com -template-kid.com -testingads.pro -thebitcoincode.com +svnuppsalaorebro.se +svolze.com +swimpool.ca +taaaak.com +tarad.com +tasteidea.com +taxi-v-eisk.ru +telsis.com +thecoolimages.net theendivechronicles.com -thetardistimes.ovh thewomenlife.com -thin.me.pn -tizanidineduringpregnancy.blogspot.com -tizanidineforopiatewithdrawal.blogspot.com -tizanidinepills.blogspot.com -tmm-kurs.ru +timeallnews.ru +titslove.yopoint.in +tizanidine4mgprice.blogspot.com +tizanidine4mgstreetprice.blogspot.com +tizanidinehcl2mgtablet.blogspot.com +tizanidineinteractions.blogspot.com +tizanidinestreetprice.blogspot.com +tkanorganizma.ru +top-karkas.ru +top10-online-games.com +top250movies.ru +toproadrunner5.info +torrentgamer.net +torrents.cd +totu.info tovaroboom.vast.ru -traffic2cash.org -trafficgenius.xyz -tripper.de -ttrraacckkrr.com -tubeline.biz -tula.mdverey.ru -twinderbella.com -tzritel.tk -ublaze.ru -udsgame.online -uhodzalijami.ru -ukrup.com -umekana.ru -uptime-as.net -urengoy.pro -url-img.link -usadacha.net -valoresito.com -vancleefreplica.pw +tracksz.co +trafficjunky.net +traxdom.ru +trion.od.ua +try-rx.com +turist-strani.ru +turizmus.us +tx41tclega.ru +ua.tc +ukrtvory.in.ua +unblocksit.es +unionmarkt.de +uzporno.mobi +vacuumscleaner.com vasileostrovsky-rayon.ru -video-camer.com -videochat.world -videokrik.net -vietimgy.pw -vikistars.com -vinsit.ru -vip.51.la -vipsiterip.org -virtualbb.com -visitcambridge.org -vkmusics.ru -vroze.com -vsdshnik.com -vykupavto-krasnodar.ru -w3javascript.com +vedomstvo.net +vezuviy.su +viddyoze.com +video-hollywood.ru +videooko.weebly.com +villacoloniale.com +virtuagirl.com +vladimir.zrus.org +vod.com.ua +voloslove.ru +vse-pesni.com +vsesubwaysurfers.com +vuclip.com +wallpaperstock.net +walpaperlist.com warezaccess.com -webix.me -websites-reviews.com -welcomeauto.ru -wetgames.ru -wingsofrefuge.net -wowas31.ucoz.ru -wowcasinoonline.ooo -woweb.com.ua -x-musics.com -xaylapdiendanang.com -xblog.in -xblognetwork.com -xgames-04.com -xjlottery.com -xjrul.com -xn-----6kcaacnblni5c5bicdpcmficy.xn--p1ai -xn----7sbifcamovvfggw9d.xn--p1ai -xn----8sbarihbihxpxqgaf0g1e.xn--80adxhks -xn----8sbdbjgb1ap7a9c4czbh.xn--p1acf -xn----8sbhefaln6acifdaon5c6f4axh.xn--p1ai -xn----btbdvdh4aafrfciljm6k.xn--p1ai -xn--80ajbshivpvn2i.xn--p1ai -xn--80ak6aa92e.com -xn--b1addnj3cah.xn--p1ai -xn--b1ag5cfn.xn--p1ai -xn--e1aggki3c.xn--80adxhks +webads.co.nz +webscouter.net +whatistizanidine2mg.blogspot.com +while.cheapwebsitehoster.com +winwotgold.pl +withstandingheartwarming.com +writingservices17.blogspot.ru +ws.ampower.me +ww2awards.info +www.jbetting.com +x-true.info +xbaboon.com +xingzi-vision.com +xmnb.net +xmronta.com +xn--b1agm2d.net +xn--d1abj0abs9d.in.ua +xn--h1ahbi.com.ua xnxx699.com -xsion.net xuki.us -xxxhdvideo.site -youdao.com -youjizz.com -youporn-ru.com -youresponsive.com -yunque.pluto.ro -zaapplesales.blogspot.com -zambini.ru -zarenica.net -zeroredirect1.com -zeroredirect12.com -zhacker.net -zhuravlev.info -zionstar.net -zmoda.hostreo.com -zona-aqua.ru -zoogdiesney.com -zrelaya.pw +xxxasianporn.net +xzlive.com +yginekologa.com +youbrainboost.asia +ypmuseum.ru +yuarra.pluto.ro +zazagames.org +zeroredirect6.com +zhorapankratov7.blogspot.com +zoompegs.com +zophim.me diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index 09553032e..86a038cb3 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -101lesbian.xyz -101raccoon.ru -108shot.com -1adult.com -214jaluwobapef.cf -51unlim.ru -777-club.ru -8si.ru -activepr.ru -acunetix-referrer.com -adidas.frwebs.fr -adsref.men +2015god.org +2345.com +2daytrendingnews.com +2girls1cup-free.com +55wmz.ru +7fon.ru +7makemoneyonline.com +7wind.ru +a.frcls.fr +a96527gi.beget.tech +aaronabel.com +adcash.com +adf.ly +adtology.com adultfullhd.com -advancedmassagebysara.com -affordablewebsitesandmobileapps.com -akusherok.ru -allfinweb.com -alltheviews.com -alvtank.se -amazon-seo-service.com -amt-k.ru -anapa-inns.ru -antons-transporte.de -apispringsmartne-a.akamaihd.net -apps-analytics.net -asscenihotosidea.blogspot.co.za -audiobangout.com -automobile-spec.com -avek.ru +aimayangzhi.com +alienwheel.es +alpharma.net +an-donut.com +anal-acrobats.com +apiwebwebgetcom-a.akamaihd.net +arewater.com +asrv-a.akamoihd.net +atlasvkusov.ru +atmagroup.ru +audiofree.ru +autochoixspinelli.com azartmix.com -banki76.ru -battle.net -beachtoday.ru -beauty-bracelet.com -bestwebsitesawards.com -beztuberkuleza.ru -bibys.com -bizcheapjerseyswholesalechina.com -blagovest-med.ru -braindaily.xyz -brandimensions.com -bristolhotel.com.ua -bvlgaribracelet.xyz -callawaygolfstore.online -canzoni.ru -capacitacionyaprendizaje.com -cashonads.com -cdncash.com -cenokos.ru -club-samodelkin.ru -co.lumb.co -cometorussia.net -cookie-law-enforcement-ff.xyz -custodieva.ru -cyse.tk -dafatiri.com -davidovic.info -dealighted.com -dickssportinggoods.com -dikqlyremy.info -discovertreasure-a.akamaihd.net -downloads-whatsapp.com +bankiem.pl +bankmib.ru +bausparen.bz.it +beachfix.co +bigames.online +bioca.org +bjetjt.com +bmw.afora.ru +bobinoz.com +brandbucket.com +browsepulse-a.akamaihd.net +callcafe.info +callstevens.com +carrauterie.be +cfcl.co.uk +chatroulette.life +chee-by.biz +coolgramgoods.com +costablue.xyz +country-chic.ru +customboxes4less.com +destinationrealestate.com +detskie-zabavi.ru +dfwu1019.info +div.as +diy-handmade-ideas.com +djekxa.ru +dollartree.info +domcran.net +downloaddy.net drugstoreforyou.com -e-collantes.com -e-commerce-seo1.com -egovaleo.it -eldiariodeguadalajara.com -elkacentr.ru -englishdictionaryfree.com -everytalk.tv -extstat.com -ezigarettekaufen2.mpbloggar.se -farmprofi.net -fashion-mk.net -fatfasts-4tmz.com -fbdownloader.com -fermersovet.ru -flowwwers.com -flytourisme.org -fm-upgrade.ru -fortevidyoze.net -forum.poker4life.ru -free-fb-traffic.com -freeuploader.ml -gael-s.ru -gamezblox.com -gay.adultgalls.com -gg.zzyjxs.com -glavtral.ru -gq-catalog.gq -gsbs.com.ua -gstatey.net -guitar-master.org -habermetre.com -hdimagegallery.net -hdxnxxtube.mobi -herokuapp.com -himazin.info -homedo.fabpage.com -hundejo.com -idc.com.ua +dscaas.website +ebonyporn.site +elektrischezigarette1.blog.pl +elektrozigarettekaufen2.blogse.nl +eupornstar.info +euromasterclass.ru +fable.in.ua +fanrto.com +farm26.ru +findacheaplawyers.com +forensicpsychiatry.ru +forex21.ru +freecamdollars.com +fz139.ttk.ru +galaxys6manual.info +gateway.zscalertwo.net +girlspicsa.com +glastecfilms.com.my +goodhumor24.com +goosefishpost.bid +greamimgo.pw +hcate.com +headpress.ru +hetmanship.xyz +hkladys.com +hoverboard360.de +i-hobot.ru +iccornacircri.cf +imagerydatabase.com +images-graphics-pics.com +img.wallpaperstock.net importchinacoach-teach.com -infogame.name -insomniagamingfestival.com -internetartfair.com -ipostroika.ru -irunfar.com -james13prix.info -jerseysbizwholesalecheap.com -joy-penguin.com -jrcigars.com -kashubadesign.ru -keywordglobal.co.za -keywordsuggest.org -kit-opt.ru -kol-energo.ru -ktotut.net -labvis.host.sk -lacloop.info -lanasshop.ru -lancheck.net -laudit.ru -levaquin750mg.blogspot.com -lifehacĸer.com -limtu.ifmo.ru -lincolntheatre.com -live-sexcam.tk -livingroomdecor.info -maridan.com.ua -matb3aa.com -megaindex.ru -michaelkorsoutlet.store -minecraft-neo.ru -missis.top -mixx.com -mnogabukaff.net -moncleroutletonline.win -mountainstream.ms -mttwtrack.com -mygameplus.com -myliveblog.ru -nash-krym.info -naturalshair.site -neks.info -netanalytics.xyz -nikitsyringedrivelg.pen.io -nucia.biz.ly -obsessionphrases.com -ohmyrings.com -online247.ml -oralsexfilme.net -orenburg-gsm.ru -paccohichetoti.ml -partnersafe.men -pcboa.se -perfectpracticeweb.com -pics-group.com -pictures-and-images.com -pinstake.com -popunder.ru -porno-play.net -pornophoto.xyz -prizefestival.mobi -pro-tec.kz -profolan.pl -prokotov.com -promalp-universal.ru -prostitutki-almata.org -purchasepillsnorx.com -pushdata.sendpulse.com -radiodigital.co -real-time-analytics.com +impresagaia.it +interfucks.net +iomoio.net +islandminingsupply.wordpress.com +jaxcube.info +jerseyssportsshop.com +karaganda.xkaz.org +kasino-money.pw +katushka.net +knogg.net +lechtaczka.net +lenvred.org +lescinq.com +lflash.ru +linksharingt.com +littlecity.ch +llastbuy.ru +lumb.co +magazin-pics.ru +mebelcomplekt.ru +medicine-4u.org +megagrabber.ru +mesto-x.com +misswell.net +mobplayer.net +mojaocena.com +monclercoats.xyz +moviemail-online.co.uk +moz.com +muschisexbilder.com +music.utrolive.ru +musicdaddy.net +mustwineblog.com +mxgetcode.com +my-aladin.com +myra.top +net-radar.com +nina.az +noscrapleftbehind.co +november-lax.com +nowtorrents.com +nudejapan.net +o-dachnik.ru +olympescort.com +onblastblog.online +onclickpredictiv.com +onemactrckr.com +oranga.host.sk +partner-stop.men +pateaswing.com +picsfair.com +pk-services.ru +pl-vouchers.com +platesauto.com +porno.simple-image.com.ua +pornoinn.com +pornosee.info +pornstartits.xblog.in +pp-budpostach.com.ua +preg.marketingvici.com +pricheski-video.com +proftests.net +prointer.net.ua +prostitutki-odessa.org +proxyelite.biz +psa48.ru +purplestats.com +pyramidlitho.webs.com +quick-seeker.com +qwarckoine.com +rankexperience.com +remontbiz.ru remontgruzovik.ru -rff-cfal.info -rixpix.ru -sashagreyblog.ga -saveindex.xyz -sawin.beth.webd.pl -scanner-donald.top -scanner-fred.top +rocis.site +rockprogblog.com +roll123.com +royal-betting.net +ruclicks.com +rusenvironmental.net +rvi.biz +sadaholding.com +saneyes.com +sanidumps.com scanner-jack.top -scanner-margo.top -scanner-mary.top -scat.porn -searchimpression.com -searchwebknow-a.akamaihd.net -security60-e.com -selectads.men -seo18.su -seobility.net -seofirmreviewsus.info -seojokes.net -sex-tracker.com -sexkontakte-seite.com -sexvideo-sex.com -simple-share-buttons.com -skatestick.bid -sluganarodu.ru -social-buttons.xyz -sockshares.tv -soda.media -solution4u.com -spywarespy.com -streamin.to -surfbuyermac.com -swinon.site -symphonyintegratedhealthcare.com -telemetryverification.net -texbaza.by -theprofitsmaker.net -thetattoohut.com -titangel-vietnam.com -tizanidinedrugtest.blogspot.com -tizanidineinfo.blogspot.com -tizanidinesleepaid.blogspot.com -tizanidinestreetprice.blogspot.com -tizanidinezanaflex.blogspot.com -top-deal.com.pl -top-study.work -top10-way.com -track2.shop -tracxn.com -truck-rece.life -tube8.com -uhodzalijami.ru -vchulkax.com -veselokloun.ru -viktoria-center.ru -voluumtracker1.com -vzlomsn.org -vzlomtw.com -wallpapersdesk.info -warezkeeper.com -webcamtalk.net -webradiology.ru -website-speed-up.top -weddingdresses.xyz -x-porno.video -xingzi-vision.com -xn--80aeahghtf8ac5i.xn--p1ai -xnxxandxvideos.com -xxlargepop.com -xxxtubesafari.com -youjizz.com -youtube-downloader.savetubevideo.com -zeroredirect6.com -zog.link -zoodrawings.com -zooggames.com -zoolubimets.ru +scanner-nelson.top +sdelatmebel.ru +seksvideoonlain.com +sexiporno.net +shitting.pro +shop.acim.org +shopfishing.com.ua +sildenafilcitratemed.com +sims-sims.ru +siteaero.com +sitechecker.pro +sivs.ru +sledstvie-veli.net +slonechka.ru +socas.pluto.ro +socialmadesimple.com +soserfis.com +soundfrost.org +spb.afora.ru +spidtest.space +spymac.net +start.myplaycity.com +stockquotes.wooeb.com +stroilka.info +strongsignal-a.akamaihd.net +studiofmp.com +stuff-about-money.com +svolze.com +tattooha.com +tedxrj.com +teenfuck.tv +themestotal.com +thepantonpractice.co.uk +tilido.com +todohr.com +top-karkas.ru +torrentgamer.net +torrnada.ru +tovaroboom.vast.ru +tvteleport.ru +twojebook.pl +tx41tclega.ru +uchetunet.su +ugogo.info +ukkala.xyz +um-razum.ru +underthesite.com +vektorpress.ru +viandpet.com +vietimgy.pw +vitanail.ru +viteonlusarezzo.it +vmnmvzsmn.over-blog.com +volgograd.xrus.org +vostoktrade.info +w-journal.ru +wallpapersist.com +webenlace.com.ar +witclub.info +womama.ru +works.if.ua +worldlovers.ru +xaijo.com +xcc24.pl +xn----dtbndd4ae7eub.top +xn--80ab4aa2g.xn--p1ai +xn--80atua3d.xn--p1ai +xn--90acenikpebbdd4f6d.xn--p1ai +yadro.ru +youporn-ru.com +youradulthosting.com +zaidia.xhost.ro +zazagames.org +zed21.net +zx6.ru diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index 981829a62..86cdae11d 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.90 0; +10.1.0.168 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index b0bf12aa1..2627cce1c 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -22462,3 +22462,1251 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2469 +### Updated: Tue Jul 13 14:17:01 UTC 2021 +### Bad Bot Count: 615 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: OPPO A33 +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ + diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index b0bf12aa1..2627cce1c 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -22462,3 +22462,1251 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2469 +### Updated: Tue Jul 13 14:17:01 UTC 2021 +### Bad Bot Count: 615 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: OPPO A33 +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ +