From db2a78c2fddd1c6f0646e3bac6b0fef5db588963 Mon Sep 17 00:00:00 2001 From: mitchellkrogza Date: Mon, 5 Jul 2021 13:17:10 +0000 Subject: [PATCH] V4.2021.07.2459 [ci skip] --- AUTO-CONFIGURATION.md | 2 +- MANUAL-CONFIGURATION.md | 2 +- README.md | 2 +- _google_analytics_ghost_spam/README.md | 2 +- .../etc/nginx/conf.d/globalblacklist.conf | 4 +- conf.d/globalblacklist-testing-version.conf | 4 +- conf.d/globalblacklist.conf | 4 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- dev-tools/buildnumber | 2 +- dev-tools/globalblacklist-testing.template | 4 +- dev-tools/globalblacklist.template | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- dev-tools/test_units/blacklist-ips.conf | 2 +- .../random-bots-for-test-quick.list | 20 +- .../test_units/random-bots-for-test.list | 320 ++--- .../random-bots-for-whitelist-test.list | 290 ++-- .../random-referrers-for-test-quick.list | 20 +- .../test_units/random-referrers-for-test.list | 936 ++++++------- .../random-referrers-for-whitelist-test.list | 484 +++---- dev-tools/test_units/whitelist-ips.conf | 2 +- dev-tools/tmp/robots.txt | 1248 +++++++++++++++++ robots.txt/robots.txt | 1248 +++++++++++++++++ 45 files changed, 3583 insertions(+), 1087 deletions(-) diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index c6c18ed42..c3cd6a6da 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2458 +#### Version: V4.2021.07.2459 #### Bad Referrers Blocked: [7066](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index ca6f4dede..7f6d85e2f 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2458 +#### Version: V4.2021.07.2459 #### Bad Referrers Blocked: [7066](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/README.md b/README.md index 68b1711eb..9fd519bdf 100755 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2021.07.2458 +#### Version: V4.2021.07.2459 #### Bad Referrers Blocked: [7066](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 64c945876..6640313ce 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,7 +3,7 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2021.07.2458 +#### Version: V4.2021.07.2459 #### Bad Referrers Blocked: [7066](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index f5da48543..ebd22856b 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2458 -### Updated: Mon Jul 5 12:52:42 UTC 2021 +### Version: V4.2021.07.2459 +### Updated: Mon Jul 5 13:11:20 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 012b34b11..901753347 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2458 -### Updated: Mon Jul 5 12:52:43 UTC 2021 +### Version: V3.2021.07.2459 +### Updated: Mon Jul 5 13:11:20 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index f5da48543..ebd22856b 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2458 -### Updated: Mon Jul 5 12:52:42 UTC 2021 +### Version: V4.2021.07.2459 +### Updated: Mon Jul 5 13:11:20 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 012b34b11..901753347 100644 --- a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2458 -### Updated: Mon Jul 5 12:52:43 UTC 2021 +### Version: V3.2021.07.2459 +### Updated: Mon Jul 5 13:11:20 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index fef092aba..306997987 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.97 1; +10.1.0.63 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index bce12f0ea..3c040781d 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.97 0; +10.1.0.63 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 012b34b11..901753347 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2458 -### Updated: Mon Jul 5 12:52:43 UTC 2021 +### Version: V3.2021.07.2459 +### Updated: Mon Jul 5 13:11:20 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index fef092aba..306997987 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.97 1; +10.1.0.63 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index bce12f0ea..3c040781d 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.97 0; +10.1.0.63 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 012b34b11..901753347 100644 --- a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2458 -### Updated: Mon Jul 5 12:52:43 UTC 2021 +### Version: V3.2021.07.2459 +### Updated: Mon Jul 5 13:11:20 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index fef092aba..306997987 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.97 1; +10.1.0.63 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index bce12f0ea..3c040781d 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.97 0; +10.1.0.63 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 012b34b11..901753347 100644 --- a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2458 -### Updated: Mon Jul 5 12:52:43 UTC 2021 +### Version: V3.2021.07.2459 +### Updated: Mon Jul 5 13:11:20 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index 0af87fa88..4edaf066b 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -2458 +2459 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index 012b34b11..901753347 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2458 -### Updated: Mon Jul 5 12:52:43 UTC 2021 +### Version: V3.2021.07.2459 +### Updated: Mon Jul 5 13:11:20 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index f5da48543..ebd22856b 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2458 -### Updated: Mon Jul 5 12:52:42 UTC 2021 +### Version: V4.2021.07.2459 +### Updated: Mon Jul 5 13:11:20 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 1de51fdbd..6bf0f6f3a 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -52.224.127.21 0; +20.94.204.162 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index f5da48543..ebd22856b 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2458 -### Updated: Mon Jul 5 12:52:42 UTC 2021 +### Version: V4.2021.07.2459 +### Updated: Mon Jul 5 13:11:20 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index fef092aba..306997987 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.97 1; +10.1.0.63 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index bce12f0ea..3c040781d 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.97 0; +10.1.0.63 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index f5da48543..ebd22856b 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2458 -### Updated: Mon Jul 5 12:52:42 UTC 2021 +### Version: V4.2021.07.2459 +### Updated: Mon Jul 5 13:11:20 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index fef092aba..306997987 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.97 1; +10.1.0.63 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index bce12f0ea..3c040781d 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.97 0; +10.1.0.63 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index f5da48543..ebd22856b 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2458 -### Updated: Mon Jul 5 12:52:42 UTC 2021 +### Version: V4.2021.07.2459 +### Updated: Mon Jul 5 13:11:20 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index fef092aba..306997987 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.97 1; +10.1.0.63 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index bce12f0ea..3c040781d 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.97 0; +10.1.0.63 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index f5da48543..ebd22856b 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2458 -### Updated: Mon Jul 5 12:52:42 UTC 2021 +### Version: V4.2021.07.2459 +### Updated: Mon Jul 5 13:11:20 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 1de51fdbd..6bf0f6f3a 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -52.224.127.21 0; +20.94.204.162 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 60b0d692a..d2f15d8f4 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2458 -### Updated: Mon Jul 5 12:52:42 UTC 2021 +### Version: V4.2021.07.2459 +### Updated: Mon Jul 5 13:11:20 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 1de51fdbd..6bf0f6f3a 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -52.224.127.21 0; +20.94.204.162 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index eff6bc3d5..b271e4b4f 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2458 -### Updated: Mon Jul 5 12:52:42 UTC 2021 +### Version: V4.2021.07.2459 +### Updated: Mon Jul 5 13:11:20 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index fef092aba..306997987 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.97 1; +10.1.0.63 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index daf36a100..1c6f46d6e 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -CheTeam -DIIbot -FrontPage -IDBot -PictureFinder -Thumbor -WPScan -Who.is Bot -mediawords -webmeup-crawler +FyberSpider +Grafula +HMView +Image Fetch +Mb2345Browser +RocketCrawler +Screaming +WEBDAV +Website Quester +meanpathbot diff --git a/dev-tools/test_units/random-bots-for-test.list b/dev-tools/test_units/random-bots-for-test.list index c06830cc5..7caa721c5 100644 --- a/dev-tools/test_units/random-bots-for-test.list +++ b/dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -01h4x.com -404checker -404enemy -ASPSeek -Abonti +Aboundexbot AhrefsBot -Aipbot -Alexibot -AlphaBot -Anarchy +Anarchie Ankit -Apexoo -Aspiegel +Anthill Asterias Attach +AwarioSmartBot +BBBike BLEXBot BackDoorBot -BacklinkCrawler -Badass -Bandit -BatchFTP -Bigfoot -BlackWidow -Bolt -Brandprotect -Brandwatch -BuiltWith +BackWeb +BlowFish +Boardreader +Buddy BuzzSumo -Calculon -Cogentbot -Copier +CODE87 +CSHttp +CazoodleBot +Cegbfeieh +CensysInspect +CherryPicker +ChinaClaw +Cocolyzebot +Collector Copyscape +Cosmos Craftbot -CrunchBot +CrazyWebCrawler Custo -DBLBot -DIIbot -DTS Agent +DSearch DatabaseDriverMysqli Devil Digincore -DigitalPebble -Discobot -DomainAppender +Discoverybot +Dispatch +DomCopBot DomainCrawler +DomainStatsBot +Domains Project Dotbot +Download Wonder +Drip ECCP/1.0 -EMail Siphon +EMail Wolf Ecxi -ExtLinksBot +Evil +Exabot ExtractorPro -EyeNetIE -FDM +Extreme Picture Finder Fimap FlashGet -Foobot -FrontPage +Flunky +Fyrebot +GT::WWW +Genieo GermCrawler GetRight GetWeb -Getintent -Go!Zilla -Gotit +Gigabot +GoZilla GrabNet Grabber Grafula -HMView +HTMLparser Haansoft -HaosouSpider Harvest -Havij -IDBot -Id-search -InfoTekies -Intelliseek -Internet Ninja -Iskanie -JamesBOT +Hloader +Humanlinks +HybridBot +IDBTE4M +IRLbot +Image Fetch +InterGET +Iria +IstellaBot +JOC Web Spider Jbrofuzz -JustView +JennyBot +JetCar +Joomla +Jorgee +Jyxobot Kenjin Spider Kinza -Lanshanbot +LNSpiderguy Larbin -LeechFTP +Lftp +LibWeb Libwhisker -LinkScan -LinkWalker +LieBaoFast +Lightspeedsystems +Linkbot Linkdexbot +LinkextractorPro LinksManager -LinqiaMetadataDownloaderBot -LinqiaScrapeBot -Lipperhey Lipperhey Spider -MIDown tool -MIIxpc +MFC_Tear_Sample +MJ12bot +MQQBrowser MSFrontPage +MSIECrawler +MTRobot +Magnet +Mail.RU_Bot Majestic-SEO -MarkWatch -Mass Downloader -Masscan +Majestic12 Mata Hari -Mb2345Browser MeanPath Bot -MegaIndex.ru -Metauri -MicroMessenger Microsoft Data Access -Microsoft URL Control Moblie Safari Mojolicious Mr.4x3 -Musobot -NICErsPRO -Nameprotect +NPbot +Name Intelligence Navroad NearSite -Needle -NetLyzer -NetSpider -NetZIP +Net Vampire +NetAnts +NetMechanic Netcraft -Nettrack +Netvibes +Nibbler +Niki-bot Nikto -Nimbostratus -Ninja Nuclei -OPPO A33 +Nutch Offline Explorer -Offline Navigator -OnCrawl +OpenLinkProfiler +OpenVAS Openfind -Openvas OrangeBot -OrangeSpider OutclicksBot +PECL::HTTP +POE-Component-Client-HTTP +Page Analyzer PageAnalyzer PageScorer -PageThing.com -Pandalytics -Panscient -Pavuk -PeoplePal -Picscout +Petalbot Picsearch +Pimonster +Pixray PleaseCrawl -ProWebWalker -Probethenet +Psbot +Pu_iN +PyCurl QueryN Metasearch -Quick-Crawler +RankActive +RankFlex +Rankivabot RankurBot -RealDownload -Ripper -RocketCrawler -SISTRIX +Reaper +Recorder +RedesScrapy +SEOkicks +SEOlyticsCrawler SMTBot +SalesIntelligent ScanAlert -Scanbot -ScoutJet +Scrapy +ScreenerBot ScrepyBot Searchestate -SearchmetricsBot Semrush -SentiBot -Shodan -Siphon +SemrushBot +Site Sucker SiteExplorer -Snapbot -Snoopy -Sociscraper -Sogou web spider -SpaceBison +SiteLockSpider +Sitebeam +Siteimprove +SlySearch +Snake Spanner -Sucker +Spinn3r +SputnikBot +Sqlworm +Sucuri +SuperBot +Surfbot Suzuran -Szukacz +Swiftbot T0PHackTeam Teleport -TeleportPro +Telesoft Telesphoreo -Telesphorep -Toata -Toweyabot -Tracemyfile -Trendictionbot +TheNomad +Thumbor +Titan +Trendiction +True_Robot +TurnitinBot +Twice Typhoeus +URLy Warning +UnisterBot Upflow +V-BOT VB Project -Vagabondo -VidibleScraper -Voil -Voltron -WASALive-Bot -WEBDAV +Vacuum +VeriCiteCrawler +Virusdie +WBSearchBot WISENutbot WPScan WWW-Collector-E -WWW::Mechanize -Wallpapers/3.0 -WallpapersHD -Web Fuck -Web Pix -Web Sucker +WWW-Mechanize +WWWOFFLE +Web Auto +Web Collage +Web Enhancer +Web Fetch WebAuto -WebFetch +WebBandit +WebCopier +WebGo IS +WebImageCollector WebReaper -WebStripper +WebSauger WebSucker WebWhacker -Webalta -WebmasterWorldForumBot -Webshag -Website Quester -WebsiteExtractor -Webster -Who.is Bot +WebsiteQuester +Whack +Widow Wonderbot Woobot -Xaldon WebSpider -YoudaoBot -Zade -ZoomBot -ZoominfoBot -ZyBorg +Wotbox +Zermelo +Zeus +Zitebot +ZmEu autoemailspider -backlink-check -com.plumanalytics -crawler4j +check1.exe +coccocbot-web +crawler.feedback demandbase-bot -domainsproject.org eCatch +facebookscraper +isitwp.com lwp-request -lwp-trivial -netEstate NE Crawler +magpie-crawler +meanpathbot +mediawords +muhstik-scan oBot page scorer -pcBrowser -plumanalytics -polaris version probe-image-size -seobility -seoscanners -serpstatbot -sexsearcher -sitechecker.pro -spyfu +scan.lol +seostar sysscan -voyagerx.com -webmeup-crawler -xpymep1.exe +zauba.io diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 0a9a00047..90ee84b43 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -360Spider +01h4x.com 404checker -ADmantX +ASPSeek Aboundex -AhrefsBot -AiHitBot +Aboundexbot +Aipbot Alexibot AllSubmitter -Alligator -Apexoo -Aspiegel +AlphaBot +Anarchie +Anarchy +Ankit Attach +AwarioRssBot +AwarioSmartBot +BDCbot BDFetch -BLEXBot +Backlink-Ceck BacklinkCrawler -Badass -BetaBot -BlowFish +Barkrowler +BatchFTP +Bitacle +BlackWidow +Blow +Boardreader Bolt -BotALot -Brandprotect Buck Buddy -BuiltBotTough -BuiltWith -BuzzSumo -CATExplorador CCBot CODE87 CSHttp -Calculon -ChinaClaw -Copyscape +CazoodleBot +CensysInspect +CherryPicker +Claritybot +Cogentbot +Cosmos Craftbot -CrazyWebCrawler Crescent +CrunchBot Curious CyotekWebCopy -DSearch +DBLBot DTS Agent DataCha0s +Devil +Digincore DigitalPebble +Disco Discobot -DittoSpyder -DomainCrawler +Discoverybot +Dispatch DomainSigmaCrawler +DomainStatsBot Domains Project +Dotbot Download Wonder +EMail Siphon +Ecxi +EroCrawler Evil Express WebPictures ExtLinksBot -Extractor -ExtractorPro -Extreme Picture Finder -EyeNetIE -Ezooms -FemtosearchBot -Firefox/7.0 -Foobot -Freeuploader -G-i-g-a-b-o-t -GalaxyBot -GermCrawler -GetRight -Gigabot -Go!Zilla +FHscan +Fimap +FrontPage +Fuzz +GT::WWW Go-Ahead-Got-It -Grabber -HMView -HTTP::Lite +Grafula +GrapeFX +HTMLparser HTTrack Haansoft -Humanlinks -HybridBot -IDBot -IlseBot -Image Fetch -Image Sucker -Intelliseek +HaosouSpider +Harvest +IDBTE4M +IRLbot +Iblog +Id-search InterGET -JOC Web Spider -Jbrofuzz -JikeSpider +Iskanie +IstellaBot +JamesBOT +Jetty Jorgee +Kenjin Spider Keyword Density -Kinza -Kozmosbot +LNSpiderguy Lanshanbot -Larbin LeechGet Lftp Libwhisker LieBaoFast -Lightspeedsystems -Linkdexbot -LinkextractorPro +LinkpadBot LinksManager -LinqiaMetadataDownloaderBot -LinqiaScrapeBot Lipperhey Lipperhey Spider -Litemage_walker Lmspider -MIIxpc MJ12bot MQQBrowser +MTRobot Mag-Net -Magnet +Mail.RU_Bot +Majestic SEO Majestic-SEO -MarkMonitor -Mata Hari -Mb2345Browser +Majestic12 +Mass Downloader +MeanPath Bot Meanpathbot Mediatoolkitbot +MegaIndex.ru Metauri -Moblie Safari -Mojeek -Mojolicious +MicroMessenger +Microsoft URL Control +Mozlila +Mr.4x3 +Msrabot NICErsPRO -Navroad +Name Intelligence Nessus -NetAnts NetLyzer +NetZIP Netvibes Nibbler +Niki-bot +Nikto Ninja -Nutch +Nuclei +Octopus +Offline Explorer +OnCrawl OpenLinkProfiler Openvas -OrangeBot -OrangeSpider -OutclicksBot +OutfoxBot PECL::HTTP -PageGrabber +POE-Component-Client-HTTP +PageAnalyzer PageScorer +Pandalytics Panscient -Pavuk -Pi-Monster -Pimonster -Pockey -ProPowerBot +Papa Foto +PictureFinder +Pixray +PleaseCrawl Pu_iN Pump Quick-Crawler RSSingBot -RankActiveLinkBot -RankFlex -RankingBot +RankActive RankurBot -RealDownload -Recorder -Ripper -Rogerbot +RebelMouse +SBIder +SEOkicks SEOkicks-Robot -SISTRIX -ScanAlert -Scanbot -Scrapy -Screaming -SeoSiteCheckup -Site Sucker -SiteExplorer +SEOprofiler +SEOstats +ScreenerBot +Searchestate +SemrushBot +Seomoz SiteSnagger SiteSucker -Sitebeam -SmartDownload +Snake Snoopy -Sociscraper +SocialRankIOBot Sogou web spider -Sottopop +Spammen +Spinn3r +SputnikBot Sqlmap +Sqworm Stripper -SuperBot +Sucuri +SuperHTTP Surfbot +Suzuran Swiftbot -T0PHackTeam -T8Abot +Teleport Telesoft -The Intraformant +TheNomad TightTwatBot -Tracemyfile -Trendictionbot -TurnitinBot -Twice +Titan +True_Robot +TwengaBot +Typhoeus +URLy Warning +URLy.Warning UnisterBot -Upflow V-BOT -VCI -Vacuum -Vagabondo -VeriCiteCrawler +VB Project +VelenPublicWebCrawler +Virusdie VoidEYE -WBSearchBot +Voltron WEBDAV WISENutbot WWW-Collector-E WWW-Mechanize WWW::Mechanize -Wallpapers/3.0 -WeSEE -Web Enhancer +WWWOFFLE +WallpapersHD +Web Auto +Web Fetch Web Pix +WebAuto WebCollage -WebFetch +WebFuck WebImageCollector -WebPix -WebSauger -WebStripper -WebSucker -WebsiteExtractor +WebReaper +WebWhacker +WebZIP +WebmasterWorldForumBot WebsiteQuester Webster Whack -Whacker -Whatweb Who.is Bot +Widow +WinHTTrack Wonderbot +Woobot Wotbox -Xaldon WebSpider -Xenu +Wprecon +Xaldon_WebSpider YoudaoBot Zade -Zeus -Zitebot +Zauba +ZmEu +ZoominfoBot ZumBot ZyBorg -autoemailspider -backlink-check -coccocbot-web +arquivo.pt +cognitiveseo crawler.feedback -crawler4j -demandbase-bot -domainsproject.org -eCatch -facebookscraper -gopher -instabid -ips-agent +internetVista monitor lwp-trivial -page scorer +netEstate NE Crawler pcBrowser polaris version -satoristudio.net +s1z.ru +scan.lol +seostar +serpstatbot sexsearcher -sitechecker.pro -spyfu +siteripz +sogouspider sysscan -trendiction.com x09Mozilla x22Mozilla +zauba.io diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 529ff3146..d50c7ffdc 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -ameblo.top -bambi.ck.ua -ccbill.com -easyshoppermac.com -jerseywholesaleelitestore.com -lizace.com -sabaapress.com -stjamesschool.info -tizanidinesleepaid.blogspot.com -xn--80ab4aa2g.xn--p1ai +amigobulls.com +b2b-lounge.com +bildsuche.ru +essay-edu.biz +footballfarrago.com +gg.zzyjxs.com +hatdc.org +labafydjxa.su +pivka.xyz +site5.com diff --git a/dev-tools/test_units/random-referrers-for-test.list b/dev-tools/test_units/random-referrers-for-test.list index 16243461a..2b1fb20a5 100644 --- a/dev-tools/test_units/random-referrers-for-test.list +++ b/dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -00go.com -0c47f8422d3f.com -1millionusd.xyz -4webmasters.com -4webmasters.org +03e.info +101flag.ru +10bet.com +178evakuator178.ru +18ps.ru +1adult.com +1kinobig.ru +21h2o.com +2345.com +24x7-server-support.site +2itech.ru +34.gs +3weekdiet.com 51unlim.ru +5kstore.com 6128786.com -abbanreddy.com -abcguru.xyz -acc.eu.org -admanaerofoil.com -admeasures.com -adpremium.org -ads.tremorhub.com -adtiger.tk -advocatemsk.ru -adzerg.com -agahinameh.com -agroeconom.kz -aimayangzhi.com -alevinefesleri.com +7makemoneyonline.com +991mostfm.co.id +aaa38852e886ac4af1a3cff9b47cab6272.com +acheterviagrafr24.com +acmebtn.ml +acortarurl.es +actionnooz.com +acunetix-referrer.com +adclickservice.com +adloads.com +alarmobninsk.ru +albamargroup.com +alborzan.com +alekseevec.ru +algarveglobal.com algerianembassy.co.in -alienwheels.de -alive-ua.com -all4wap.ru -alphacarolinas.com +alif-ba-ta.com +allsilver925.co.il alphavisions.net -alvtank.se -am-se.com -anatomiy.com -android-style.com -android4fun.org -animebox.com.ua -anmysite.com -apisecretsaucebi-a.akamaihd.net -apptester.tk -aquarium-pro.ru -asophoto.com -atleticpharm.org -atmagroup.ru -autoplate.info -avtoarenda.by -avtocredit-legko.ru -ayanaspa.com -baditri.com -banki76.ru -beachdriveblog.com -beachfix.co -bensbargains.net -berrymall.ru -bestdraws.com -bestkfiledxd.cf -bestwebsiteawards.com -bgviagrachrx.com -bharatdefencekavach.com -bigpictures.club -bimmerpost.com +amateurlivechat.org +amung.us +angigreene.com +apiadanaknet-a.akamaihd.net +apibatbrowsecom-a.akamaihd.net +apiholdingmypage-a.akamaihd.net +apioasisspacenet-a.akamaihd.net +apispringsmartne-a.akamaihd.net +apple.com-cleaner.systems +arcteryxsale.online +artparquet.ru +asmxsatadriverin.aircus.com +asrv-a.akamoihd.net +atlant-auto.info +audiofree.ru +automatic-seo.com +avtointeres.ru +azartniy-bonus.com +azbaseimages.net +azinoofficial777.ru +backiee.com +balois.worldbreak.com +baoxaydung.com.vn +barnfurnituremart.com +battleforupdating.site +bbtec.net +beclean-nn.ru +big-cash.net binomo.kz -biocn.dx.am -bitcoins-live.ru -bitporno.sx -bloggedporn.com -boostmyppc.com -braindaily.xyz -bravegirlsclub.com -breget74.ru -buscarfoto.com -buttons-for-website.com -buy-cheap-pills-order-online.com -bwlx.prepedu.cn -candiceloves.us -cannibalcheerleader.com -canrole.com -carapics.com -cardsdumps.com -cartierreplica.top -cartierring.xyz -cashkitten-a.akamaihd.net -castingbank.ru -catalogs-parts.com -cementaresearch.se -centrdebut.ru -chatroulette.si -cheapjerseysap.com -cherrypointplace.ca -christianlouboutinreplica.win -cialgenisrx.com -cibpenonptib.flu.cc -clubfashionista.com -coin-hive.com -com-secure.download -compliance-george.top -compliance-irvin.top -compliance-josh.top -compliance-willy.top -contextualyield.com +bird1.ru +bitnote.co +blavia.00author.com +blockworld.ru +blogporn.in +bluerobot.info +bonux.nextview.ru +bonvillan.com +bookmark4you.com +bosefux.esy.es +braip.com.br +bristolhotel.com.ua +brothers-smaller.ru +bugof.gq +buigas.00it.com +bupropion-sr-150-mg.us +businesxxl.com +buy-forum.ru +buyk.host.sk +bvlgariring.xyz +callcafe.info +cand.jp +carstrends2015.com +cartierbraceletsreplica.pw +cartierreplica.xyz +cartiertrinity.xyz +casinorewards.info +ccjp.eu +cctva.tv +cheatcode-lita12.rhcloud.com +checkhit.com +chuckguilford.com +cialischstgerts.com +clickbanksites.info +clickcash.com +clipartnew.com +clothesforcash.com +collegerentals.com +com-supportcenter.website +communisave.co.za +connectingsingles.com +converse.ddsoldes.fr cookie-law-enforcement-aa.xyz -cosmeticswomens-womensports.rhcloud.com -covid-schutzmasken.de -cpajunkies.com -crackzplanet.com -crest3d.ru -cubook.supernew.org +copypaste.traderzplanet.in +counter.bloke.com culturevie.info -cutt.us -cvety24.by -cymbaltaandtizanidine.blogspot.com -dailyrank.net +custodieva.ru +customsua.com.ua +dailyfinancefix.com +dailystrength.org +daisye.top dalavia.ru -daneshetabiat.com -data1.scopich.com -ddpills.com -de.zapmeta.com -dealighted.com -dear-diary.ru -delfin-aqua.com.ua +dame-ns.kz +daymusam.com depositfiles-porn.ga -dieta-personalna.pl -digital-video-processing.com -dikqlyremy.info -dikx.gdn -dirtpics.pw -doctissimo.fr -doctormakarova.ru -doktoronline.no -domainsphoto.com -domashneeruporno.com -domznaniy.ru -doreenblog.online -download-of-the-warez.blogspot.com -downvids.net -dpihatinh.gov.vn -drhomes.biz -drpornogratisx.xxx -dvr.biz.ua -e-biznes.info -ebooknovel.club -editors.choice6912650.hulfingtonpost.com -ege-essay.ru -elektrischezigarette1.blog.pl -elite-sex-finders.com -emailaccountlogin.co -embedle.com -enbersoft.com -enews.tech -enpolis.ru -enthuse.computernetworksonline.com -envaseslotusama.com -eonpal.com -epicbrogaming.com -erotik-kostenlos.net -essaypro.com -estibot.com -evogarage.com -exct.net -exmasters.com -factorynightclub.com -fakehandbags.xyz -familyphysician.ru -filmi-onlain.info -find1friend.com -fingerprintjs.com -fireads.men -fitness-video.net -flashahead.info -formseo.com -formulaantiuban.com -foto-telok.net -free-social-buttons.xyz -free411games.com -freefoto.ca -freshsuperbloop.com -friendflnder.com -fuckbuddybestgilf.info +derevesendeco.com +devochki.top +dlya-android.org +documentsite.net +dogbreedspicture.net +dokfilms.net +domaineaneblanc.com +dorratex.tn +dptaughtme.com +drillsaw.ru +e-commerce-seo.com +eachdayisagift.review +eas-seo.com +elektrischezigaretten1.blogse.nl +elektronischezigarettekaufen2.tumblr.com +elvenar.com +elvenmachine.com +energy-ua.com +eupornstar.info +evaashop.ru +evehemming.blogspot.com.au +excitacion.info +expdom.com +eyelike.com.ua +ezigarettekaufen2.blox.pl +ezigarettenkaufen2.dreamwidth.org +fantasticpixcool.com +fbdownloader.com +ferieboligkbh.dk +ferretsoft.com +fetlifeblog.com +figuringmoneyout.com +finansov.info +findclan.org +findthe.pet +flash4fun.com +footbalive.org +forex.osobye.ru +fortwosmartcar.pw +forum69.info +foxjuegos.com +foxydeal.com +frameimage.org +free-traffic.xyz +freevpn.space +freza-sverlo.ru +fuck-paid-share-buttons.xyz fullgirl.ru -fun-mobi.pl -fungamelands.com -funtoonez.com -galeon.com -gallictures.com +funponsel.com +fxund.us +fyxabomiw.ru +gambarxkata.co game300.ru gamerextra.com +gamingspark.com gate5.co.za -gateway.zscalerone.net -gay-file.com -gay-site.store -geckoandfly.com -getridofstretchmarks.org -given2.com -glicol.kz -goblacked.com -gosreg.amchs.ru -graphics8.info -growmyfunds.ca -gsasearchenginerankerdiscount.com -haikuware.com -handsandlegs.ru -hd-film.pl -highland-homes.com -historichometeam.com -hollywoodactress.info -hostingclub.lk -hotelcrocenzi.sm -hotshoppymac.com -housekuba.org -houtings.xyz -id-forex.com -ihc2015.info -ihtec2019.org -imagine-ex.co -imgata.com -imguramx.pw -iminent.com -impotentik.com -incolors.club -increasewwwtraffic.info -instasexyblog.com -interesnie-faktu.ru -ipchicken.com -iplogger.org +gateway.zscalertwo.net +gdzkurokam.ru +germes-trans.com +getaclueamerica.com +gilsonchiro.xyz +girlspicsa.com +gk170.ru +glastecfilms.com.my +go2album.com +go2mike.ru +gok-kasten.net +goldenggames.com +googlemare.com +googst2.ru +gotomontenegro.net +gtopstats.com +guruofcasino.com +helicalpile.us +hermesreplica.pw +hermesreplica.win +heroz.fr +hikesearch.net +histats.com +hit-men.men +homeinns.com +homelygarden.com +hoste.octopis.com +hosting-tracker.com +hotchatdate.com +houseofrose.com +hoztorg-opt.ru +hscsscotland.com +hully.altervista.org +hyiphunter.org +i-hobot.ru +igtools.club +ilte.info +imagecoolpub.com +imk.com.ua +immobiliaremassaro.com +impisr.ru +infektsii.com +infobanks.ru +intim-uslugi.info +intimshop-fantasy.ru +iphantom.com iptool.xyz -iwantmyfreecash.com -jamiembrown.com -jmat.cn -joessmogtestonly.com -journeydownthescale.info -jpcycles.com -jrpmakati.com -kabbalah-red-bracelets.com -kakablog.net -kareliatobacco.ru -kazrent.com -kenaba.su -kinky-fetishes.com -kinoflux.net -kinomaniatv.pl -kinotorka.ru -kinozapas.org -klejonka.info +isotoner.com +ix20.ru +ixora.pro +izismile.com +j33x.com +jaxcube.info +jerseychinabizwholesale.com +jovencitas.gratis +jrcigars.com +justucalling32211123456789.tk +kakadu-interior.com.ua +kartiniresto.com +kashubadesign.ru +keywordsuggest.org +kfon.eu +kinobaks.com +kinoplen.ru kol-energo.ru -konkursowo-24.pl konturkrasoty.ru -kostenloser-sex.com -kredit-blog.ru -kvartira-sutochno.com -lacloop.info -ladov.ru -lietaer.com -liffa.ru -likesdesign.com -linkdebrideur.xyz -lion.snapmap.us -liran-locks.com -liver-chintai.org -liveu.infoteka.hu -loadingpages.me -locatellicorretor.com.br -locimge.pw -longgreen.info -louboutinbooties.xyz -louboutinreplica.xyz -love-baby.cz -luciddiagnostics.in -lutherstable.org -lyngdalhudterapi.no -mac-shield.com -machicon-akihabara.info -mactechinfo.info -magicalfind-a.akamaihd.net -magicdiet.gq -makis.nu -manifestyourmillion.com -mansparskats.com -matpre.top -meetingrainstorm.bid -meetlocalchicks.com -mejoresfotos.eu -mezaruk.info +kostenlos-sexvideos.com +kumuk.info +kupit-adenu.ru +kurbappeal.info +kw21.org +landreferat.ru +lapitec.eu +larutti.ru +lawyersinfo.org +li-er.ru +likesuccess.com +liky.co.ua +limads.men +liveinternet.ro +lmrauction.com +loadopia.com +lombia.com +lost-alpha.ru +m4ever.net +managerpak204.weebly.com +manuscript.su +matrixalchemy.com +matsdale.com +mature.free-websites.com +me-ke.com +meble-bogart.info +memberty.com +mensandals.xyz +mere.host.sk +metabar.ru +mhtr.be microsoftportal.net -minet.club -mmstat.com -mncrftpcs.com +mielec.pl +misandesign.se +mistr-x.org mobifunapp.weebly.com -mole.pluto.ro -mompussy.net -monitorwebsitespeed.com -mrbitsandbytes.com -muizre.ru -mulberryoutletonlineeu.com -music7s.me -mvpicton.co.uk -najaden.no -nalogovyykodeks.ru -naperehresti.info -narkologiya-orel.ru -narutonaruto.ru -nash-krym.info -naturalbreakthroughsresearch.com -naturalpharm.com.ua -netanalytics.xyz -networkad.net -newtechspb.ru -nikitsyringedrivelg.pen.io -nw-servis.ru -obesidadealgarve.com -obiavo.kz -oneminutesite.it -onko-24.com -onlinemeetingnow.com -paidonlinesites.com -paighambot.com -paradontozanet.ru -partner-high.men -partner-print.men -perfection-pleasure.ru +monclercheap.xyz +moroccosurfadventures.com +morpicert.pw +mriyadh.com +msfsaar.de +mtmtv.info +mundoaberrante.com +musicspire.online +muycerdas.xxx +myhydros.org +myiptest.com +mylida.org +mypets.by +mysex21.com +nationalbreakdown.com +net-profits.xyz +netpics.org +newstudio.tv +nfljerseyscheapchinabiz.com +nicovideo.jp +nippon-bearings.ru +nodding-passion.tk +normalegal.ru +nottyu.xyz +obsessionphrases.com +ok.ru +omoikiri-japan.ru +onlinebay.ru +onlineslotmaschine.com +onlinewritingjobs17.blogspot.ru +outclicks.net +page2rss.com +partner-pop.men +partybunny.ru +pcboa.se +penisvergrotendepillennl.ovh +pensplan4u.com +perfectpracticeweb.com +perso.wanadoo.es +petsblogroll.com peugeot-club.org -phuketscreen.com -pics-group.com -picurams.pw -pila.pl -pivka.xyz -pixelrz.com -platesauto.com -pogosh.com -pokemon-go-play.online -porndroids.com -pornhubforum.tk -porno-video-chati.ru -pornoelita.info -pornohub.me -pornoklad.net -pornoklad.ru -pornophoto.xyz -pornoplen.com +phobia.us +phrcialiled.com +pillscheap24h.com +pinup-casino1.ru +pixshark.com +pizdopletka.club +pizza-tycoon.com +play.leadzupc.com +pliks.pl +polcin.de +poligon.com +popunder.net +pornmania.pl +porno-play.net +pornofiljmi.com +pornosee.info +pornosmola.info +pornstartits.xblog.in pornzone.tv -porodasobak.net -porto.abuilder.net -privatefx-in.ru -privatov-zapisi.ru -prizefestival.mobi -pulse33.ru -qitt.ru -qwertty.net -rabotaetvse.ru -raisedseo.com -ranksonic.info -rapidgator-porn.ga -rapidokbrain.com +pretty-mart.com +prodess.ru +professionaldieselcare.com +prombudpostach.com.ua +promover.org +prostitutki-kharkova.org +prostitutki-kiev.org +rada.ru +rankia.com razleton.com -realizmobi.com -realresultslist.com -reckonstat.info -remontbiz.ru -remote-dba.de -repeatlogo.co.uk -respublica-otel.ru -resultshub-a.akamaihd.net -ringporno.com -rocketchange.ru -rockma.se -roma-kukareku.livejournal.com -royal-investments.net -rucrypt.com -russia-tao.ru -rvi.biz -rybalka-opt.ru -sabizonline.com +redirlock.com +rednise.com +registrydomainservices.com +remont-ustanovka-tehniki.ru +remorcicomerciale.ro +remybutler.fr +resellerclub.com +rethinkwasteni.info +reward-survey.net +rfserial.net +riralmolamsaca.tk +robot-forex.biz +rollercoin.com +royalcar-ufa.ru +rozalli.com +rvzr-a.akamaihd.net +sakhboard.ru +salutmontreal.com +sandhillsonline.com +saratov.xrus.org +sarm.tk +saugatuck.com sawin.beth.webd.pl -scanner-mary.top -scoopquest.com -scrapinghub.com -screentoolkit.com -se.bnt-team.com -seminarykansas904.ml -semrush.com -seoexperimenty.ru -seoheap.com -seoholding.com -servingnotice.com -sex-sex-sex5.com -sexkontakteao.info -sexkrasivo.net -shama-rc.net +scanner-mark.top +score-ads.men +searchmywindow-a.akamaihd.net +seksvideoonlain.com +seorankinglinks.com +servicecenter.co.ua +sex-dating.co +sexblog.pw +sexphoto.site +sexpornotales.net +sextracker.be +sexvideo-sex.com +sexystrippe.info +sexyteens.hol.es sharebutton.net -sharebutton.to -shareyards.com +sheerseo.com shemalegalls.blogporn.in -shijian.ac.cn -shitting.pro -shlyahten.ru -sideeffectsoftizanidine.blogspot.com -simple-image.com.ua -sirpornogratis.xxx -site-speed-check.site -snjack.info -snow.nvr163.com +shinikiev.com.ua +shisha-swag.de +shohanb.com +shymkent.xkaz.org +sibdevice.ru +sibecoprom.ru +simon3.ru +simple-share-buttons.com +simplesite.com +sims-sims.ru +siteexpress.co.il +snip.tw soc-econom-problems.ru -softomix.com -sogimlecal.tk -solinf.co -speeddream.xyz +social-button.xyz +softxaker.ru +soheavyblog.com +sophang8.com +spl63.fr sports-supplements.us -spywarebegone.com -spywarenuker.com -spywarespy.com -standardchartered-forex.com -starwars.wikia.com -staynplay.net -stevemonsen.com -store-rx.com +stroilka.info stroiminsk.org -talant-factory.ru -tamada69.com -tattomedia.com +studentguide.ru +studiofmp.com +stuff-about-money.com +suggest-keywords.com +supernew.org +svarkagid.com +swinger-mobil.net +tastyfoodideas.com +taximytishi.ru +technika-remont.ru +tecnoteakviareggio.it teenfuck.tv -templates.radiodigital.co -thelottosecrets.com -thewebsitetemplate.info -thruport.com -timer4web.com -tizanidinenarcotic.blogspot.com -tizanidinesideeffects.blogspot.com -toonfamilies.net -top-l2.com +thecoolimages.net +thetoiletpaper.com +tizanidineduringpregnancy.blogspot.com +tizanidinehcl2mgsideeffects.blogspot.com +tizanidinehcl4mgisitanarcotic.blogspot.com +tizanidineinteractions.blogspot.com +tizanidineusedfor.blogspot.com +tk-assortiment.ru +tmearegion26.com +top-karkas.ru top1-seo-service.com -topcar-krasnodar.ru -torrent-to-magnet.com -trackzapper.com -traffic100.com -trafficreceiver.club -trustaffs.com -tsstcorpcddvdwshbbdriverfb.aircus.com -tvory.predmety.in.ua -typer.one -ugguk.online -ukrup.com -umaseh.com -unblocksit.es -unimodemhalfduplefw.pen.io -unmaroll.ya.ru -uptime-delta.net -ural-buldozer.ru -usiad.net -ussearche.cf -valsalud.com -vasileostrovsky-rayon.ru -vellings.info -versaut.xxx-cam.webcam -vesnatehno.ru -video--production.com -video-camer.com -vipromoffers.com -virus-respirators.com -visa-pasport.ru -visionwell.com.cn -vizag.kharkov.ua -vkmusics.ru -vkonche.com -vkontaktemusic.ru -voloslove.ru -vykup-avto-krasnodar.ru -wardreapptokone.tk -webcamtalk.net -webix.biz -website-analytics.online -website-speed-check.site -welck.octopis.com -wesharepics.info -westen-z.life -whereiskentoday.com -whereverdesperate.gq +top10-online-games.com +tovaroboom.vast.ru +tramadolandtizanidine.blogspot.com +tx41tclega.ru +uchetunet.su +ultimateclassicrock.com +unitygame3d.com +upupa.net +urlopener.blogspot.com.au +uzporno.mobi +velobikestock.com +vikistars.com +virus-schutzmasken.de +visitcambridge.org +viveresaniesnelli.it +vmnmvzsmn.over-blog.com +voluumtracker1.com +vremya.eu +vroze.com +vvpg.ru +vzlomsn.org +vzlomtw.com +walkme.com +waterefficiency.co +wcb.su +webcamdevochka.com +webextract.profound.net +webix.me +welcomeauto.ru +westsextube.com +white-truck.life +whiteelephantwellington.com +wholesalecheapjerseysfree.com +wholesalejerseychinaoutlet.com wildworld.site -womama.ru -woman-tampon.ru -wtsindia.in -www.souvenirua.com -wygraj-skiny.win -x-true.info -xerox-douglas.cf -xev.ru -xlolitka.com -xn-----6kcaacnblni5c5bicdpcmficy.xn--p1ai -xn--80aaafbn2bc2ahdfrfkln6l.xn--p1ai -xn--lifehacer-1rb.com -xnxxandxvideos.com -xolodremont.ru +wineitudes.wordpress.com +wineration.com +wnhjavlhezp.gq +woman-orgasm.ru +word-vorlagen.xyz +worldoffiles.ru +www.888.com +x69ty.ru +xlovecam.com +xn-----6kcaabbafhu7cskl7akvongwpo7hvjj.xn--p1ai +xn--80ajbshivpvn2i.xn--p1ai +xn--b1adccaf1bzj.xn--p1ai xpresscare.ru -xtremeeagles.net -xvideospornoru.com -xwatt.ru -yellocloud.be -yuarra.pluto.ro -z-master.ru -zazagames.org -zero1.it -zeroredirect10.com -znakomstvaonlain.ru +xxxsiterips.xyz +youbrainboost.asia +yougotanewdomain.com +youradexchange.com +yourmovies.pl +yourserverisdown.com +ypmuseum.ru +zagadki.in.ua +zaloadi.ru +zaloro.com +zeg-distribution.com +zeroredirect2.com +zeroredirect7.com +zhuravlev.info +zigarettenonlinekaufen2.bloog.pl +zionstar.net +zirondelli.it znaturaloriginal.com -zvuker.net -zzlgxh.com +zona-aqua.ru +zoogdinsney.com +zootravel.com diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index bfff4620b..1e96e70fb 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -404.mn -aa625d84f1587749c1ab011d6f269f7d64.com +100searchengines.com +178evakuator178.ru +4rent.ru +4ureyesonly.com +5forex.ru +6hopping.com +9icmzvn6.website +aa81bf391151884adfa3dd677e41f94be1.com aa9bd78f328a6a41279d0fad0a88df1901.com -abclauncher.com -adobereader-free.ru -agapovdg.ru -agreda.pluto.ro -aimayangzhi.com -alldubai.biz -anal-acrobats.hol.es -angigreene.com -apartmentbay.ru -apessay.com -apibetweenlinesn-a.akamaihd.net -apiitsthirteende-a.akamaihd.net -apimossnetinfo-a.akamaihd.net -apiwoodensealcom-a.akamaihd.net -asdfz.pro -avtochehli.by -avtocredit-legko.ru -beauty-bracelet.com -bestmarriages.com -bestpornuha.com -blackplanet.com -bobinoz.com -bocoarchives.org -boleznikogi.com -brokergid.ru -capture-room.com -casite-513049.cloudaccess.net -cheapjerseysap.com +aadroid.net +acmebtn.ml +adinterax.com +adtech.fr +albuteroli.com +algarveglobal.com +allkrim.com +amt-k.ru +approvedlocal.co.za +arendas.net +asscenihotosidea.blogspot.co.za +atmagroup.ru +atmovs.com +autochoixspinelli.com +automatic-seo.com +autoua.top +azartmix.com +babespcs.com +baikaleminer.com +beachtoday.ru +beclean-nn.ru +berdasovivan.ru +bestssaker.com +betune.onlinewebshop.net +binomo.kz +bitcoinpile.com +bjanshee.ru +brainboosting.club +brothers-smaller.ru +bum.com.ru +businescoop.men +buy-meds24.com +buyantiviralwp.com +callawaygolfoutlet.online +capacitacionyaprendizaje.com +capsderfudd.tk +cartierloveringreplica.xyz +cdncash.net +centrdebut.ru +chatroulette.si +cheap-pills-norx.com +cheapestjerseys-wholesale.com chocoslim.pro -cjmc.info -clickcash.com -clips.ua.ac.be -compliance-irvin.top -compliance-mary.top -compliance-olga.top -converse.ddsoldes.fr -customchocolate.business-for-home.com -cyse.tk -delayreferat.ru -dfiles.me -diamond-necklace.info +cialiswithoutadoctor.net +citetick.com +cleanmypc.com +clicksor.net +cocolyze.com +com-cleaner.systems +compliance-fred.top +concordexoticrentals.com +contentlook.co +contentsexpress.com +cowblog.fr +d-black.bz +d0t.ru +dailystrength.org +danceuniverse.ru dieswaene.com -doska-vsem.ru -drivotracker.com -e-commerce-seo1.com -easyshoppermac.com -editors.choice6912650.hulfingtonpost.com -elektrischezigarette1.blog.pl -elektrozigarettekaufen2.blogse.nl -encodable.com -erotik0049.com -eupornstar.info -ezigarettekaufen1.hpage.com -ezigarettekaufen2.yolasite.com -ezrvrentals.com -f0815.de -fashion-stickers.ru -fatmaelgarny.com -fillmewithhappiness.com -findthe.pet -food.dtu.dk -forums.d2jsp.org -freetangodownload.com -frivgame250.com -frvo.alptandem.ru -fuck-paid-share-buttons.xyz -fun-mobi.pl -funnypica.com -gallictures.com -gdebestkupit.ru -gemgrab-a.akamaihd.net -generic-pills-online.com -getlaid-xxxhookupdirect.com -gheus.altervista.org -ghostvisitor.com -gk170.ru -goodprotein.ru -gopro-online.info -gosmeb.ru -gsasearchengineranker.space -gsmlab.pl -highstairs-a.akamaihd.net -homedo.fabpage.com -hostingclub.lk -hotelcrocenzi.sm -ideashome.id -iflycapetown.co.za -imabase.com -images-free.net -infobanks.ru -investmac.com -investpamm.ru -izismile.com -jerseyschinabizwholesale.us -jovencitas.gratis -kaac.ru -kalb.ru -kaz.kz -keywordhut.com -keywordspay.com -kinky-fetishes.com -kino-filmi.com -kinobaks.com -kinocccp.net -kinotorka.ru -kirov.zrus.org -lacloop.info -languagecode.com -lapolis.it -laudit.ru -lesbian.xyz -lifebyleese.com -limtu.ifmo.ru -littlecity.ch -live-sexchat.ru -lonely-mature.com -lookingglassemb.com -mabdoola.blogspot.com -machicon-akihabara.info -mackeeper-center.club -manimpotence.com -maximilitary.ru -med-zdorovie.com.ua -medi-fitt.hu -merryhouse.co.uk -miniads.ca -missclub.info -mobplayer.net -modabutik.ru -money-for-placing-articles.com -morlat.altervista.org -mp3downloadhq.com -myprintscreen.com -myshopmatemac.com -mzdish.site -nalogovyy-kodeks.ru -nardulan.com -netvouz.com -nhl17coins.exblog.jp -nvssf.com -nw-servis.ru -oddamzadarmo.eu -of-ireland.info -offer.wpsecurity.website -onlinemegax.com -opinionreelle.com -partner-pop.men -partnersafe.men -pcimforum.com -pictures-and-images.net -pinwallpaper.top -pio.polytopesexempt.com -pipki.r.acdnpro.com -pistonclasico.com -pix24x7.com -pl-vouchers.com +dimkino.ru +doreenblog.online +dostavka-v-ukrainu.ru +downloadeer.net +dyshagi.ru +eas-seo.com +education-cz.ru +ege-essay.ru +elvel.com.ua +emergencyneeds.org +eonpal.com +escortslet.net +estelight.ru +fastcrawl.com +fenoyl.batcave.net +fetlifeblog.com +filesmonster.porn +fireads.men +fix-website-errors.com +forex.osobye.ru +forum.doctissimo.fr +free-deals.faith +fungamelands.com +generalporn.org +goldenggames.com +gonextmedia.com +groupmoney.ru +guidefs.ru +gungamesz.com +heartofpayne.xyz +hexpilot.com +hoholikik.club +hop.clickbank.net +houdom.net +humanorightswatch.org +ibmdatamanagement.co +imagefinder.site +iminent.com +imitex-plus.ru +inboxdollars.com +indo-export.ru +insurple.com +ipostroika.ru +itch.io +itrevolution.cf +itsdp3.com +iwanttodeliver.com +izhevsk.xrus.org +jerseychinabizwholesale.us +kalandranis.gr +kamagragelusa.net +kinogolos.ru +konica.kz +konturkrasoty.ru +kurbappeal.info +louboutinshoes.xyz +luckyday.world +m.b00kmarks.com +magicdiet.gq +mailemedicinals.com +marcoislandvacations.net +marketingtechniques.info +mediafresh.online +medicines-choice.com +megaindex.ru +members.ghanaweb.com +mmstat.com +moivestiy.biz +monarhs.info +moscow.online-podarki.com +movpod.in +multgo.ru +narkologiya-peterburg.ru +nightvision746.weebly.com +nisuturnetdgu.tk +nofreezingmac.work +o-o-11-o-o.com +oballergiya.ru +okout.ru +online-x.ru +onlinemeetingnow.com +onstrapon.purplesphere.in +openfrost.net +osb.se11.ru +partybunny.ru +pekori.to +piluli.info +pinwallpaper.xyz +playmsn.com +plusnetwork.com +popunder.ru pornoelita.info -pozvonim.com -pshare.biz +pornoforadult.com +pozdrawleniya.com +prchecker.info +priora-2.com +privatov-zapisi.ru +prosperent.com +prostitutki-kharkova.org +psoriasis-file.trade ptr.ruvds.com -push-ad.com -quickcashlimited.com -quizzitch.net -radiodigital.co -realizmobi.com -reklama-i-rabota.ru -remedyotc.com -replicalouboutin.xyz -resant.ru -retreatia.com -sbdl.no -sc-specialhost.com -scanner-donald.top -scenarii-1-sentyabrya.uroki.org.ua -scenicmissouri.us -se.bnt-team.com -seansonline24.pl -seeingmeerkat.com -seo-prof1.xyz -servingnotice.com -sexpornotales.net -shakhtar-doneck.ru -share-buttons-for-free.com -sideeffectsoftizanidine.blogspot.com -smstraf.ru -snip.to -soheavyblog.com -songplanet.ru -speeddream.xyz -spinnerco.ca -sprttrack.com -sptslmtrafms.com -srgwebmail.nl -stair.registrydomainservices.com -stard.shop -stauga.altervista.org +pulseonclick.com +rating-bestcasino.com +razleton.com +razyboard.com +realmonte.net +realnye-otzyvy.info +redirlock.com +reining.lovasszovetseg.hu +remont-fridge-tv.ru +russkoe-zdorovie.ru +samoiedo.it +sbricur.com +sei80.com +seksvideoonlain.com +seminarygeorgia59.ga +seo-tools-optimizing.com +seofirmreviewsus.info +sexytrend.ru +shareyards.com +shodanhq.com +shoppingmiracles.co.uk +si-unique.com +sibtest.ru +sibvitr.ru +site.ru +slkrm.ru +smartbalanceworld.com +sobecjvuwa.com.ru +socialbuttons.xyz steame.ru -stockspmb.info -stroicol.net -strv.se -sukirgenk.dvrlists.com -t-bygg.com -teencastingporn.com -tetracsaudi.com -textads.men -thegamerznetwork.com -tizanidine6mg.blogspot.com -tizanidineandhydrocodone.blogspot.com -tizanidinepills.blogspot.com -tizanidinesleepaid.blogspot.com -tmearegion26.com -torture.ml -traffique.net -twittrading.com -uhdtv.website -ukkala.xyz -ukrtextbook.com -ukrtvir.com.ua +studiokamyk.com.pl +styro.ru +suggest-keywords.com +tam-gde-more.ru +tastyfoodideas.com +telefonsexi.com +theautoprofit.ml +ticketsys.inetwd.com +tizanidine4mgstreetvalue.blogspot.com +tizanidineandgabapentin.blogspot.com +tizanidinemusclerelaxant.blogspot.com +tnctrx.com +topflownews.com +torrent-to-magnet.com +totu.info +trubywriting.com +twiclub.in +uac.net.au +udsgame.online +ukrobstep.com ultimatesetnewfreeallsoftupgradesystems.pw -upstore.me -uzporno.mobi -videochat.ph -vinylvault.co.uk -vipromoffers.com -vipsexfinders.com -vzubah.com +uni.me +uroki.net +uzungil.com +vancleefreplica.pw +vapsy.com +vekzdorov.ru +velpanex.ru +viteonlusarezzo.it +wait3sec.org +warezkeeper.com +watch-movies.ru wemarketing.se -wieseversa.no -witherrom55.eklablog.fr -wonderfulflowers.biz -worldmusicfests.com -wsgames.ru -ww2awards.info -xgames-04.com -xn----itbkqkfiq.xn--p1ai -xn--80aodinpgi.xn--p1ai -xn--h1ahbi.com.ua -xnxx-n.com -yadro.ru -ypmuseum.ru -zajm-zalog-krasnodar.ru -zastenchivosti.net -zdesformula.ru -zeikopay.com -zhorapankratov7.blogspot.com -zixizop.net.ru -zkjovpdgxivg.ga +winner7777.net +wma-x.com +wyniki-lista.pl +xn----7sbifcamovvfggw9d.xn--p1ai +xn--80abgj3a5acid6ghs.top +xvideosporn.biz +xz618.com +yoshkarola.zrus.org +youtubedownload.org +z-master.ru +za-fun-offer.com +zaloadi.ru +zaloro.com +zarabotat-na-sajte.ru +zastroyka.org +zazagames.org +zed21.net +zeroredirect2.com diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index bce12f0ea..3c040781d 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.97 0; +10.1.0.63 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index cca350596..bdd8c12bb 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -9982,3 +9982,1251 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2459 +### Updated: Mon Jul 5 13:11:21 UTC 2021 +### Bad Bot Count: 615 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: OPPO A33 +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ + diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index cca350596..bdd8c12bb 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -9982,3 +9982,1251 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2459 +### Updated: Mon Jul 5 13:11:21 UTC 2021 +### Bad Bot Count: 615 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: OPPO A33 +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ +