From 16840461eb6a113f0e3e80057e5be015a1908740 Mon Sep 17 00:00:00 2001 From: mitchellkrogza Date: Mon, 5 Jul 2021 13:32:38 +0000 Subject: [PATCH] V4.2021.07.2460 [ci skip] --- AUTO-CONFIGURATION.md | 2 +- MANUAL-CONFIGURATION.md | 2 +- README.md | 2 +- _google_analytics_ghost_spam/README.md | 2 +- .../etc/nginx/conf.d/globalblacklist.conf | 4 +- conf.d/globalblacklist-testing-version.conf | 4 +- conf.d/globalblacklist.conf | 4 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- dev-tools/buildnumber | 2 +- dev-tools/globalblacklist-testing.template | 4 +- dev-tools/globalblacklist.template | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- dev-tools/test_units/blacklist-ips.conf | 2 +- .../random-bots-for-test-quick.list | 20 +- .../test_units/random-bots-for-test.list | 302 ++-- .../random-bots-for-whitelist-test.list | 286 ++-- .../random-referrers-for-test-quick.list | 20 +- .../test_units/random-referrers-for-test.list | 932 ++++++------ .../random-referrers-for-whitelist-test.list | 486 +++---- dev-tools/test_units/ratelimittest-master.txt | 10 - dev-tools/test_units/whitelist-ips.conf | 2 +- dev-tools/tmp/robots.txt | 1248 +++++++++++++++++ robots.txt/robots.txt | 1248 +++++++++++++++++ 46 files changed, 3571 insertions(+), 1085 deletions(-) diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index c3cd6a6da..4f0242ab0 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2459 +#### Version: V4.2021.07.2460 #### Bad Referrers Blocked: [7066](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 7f6d85e2f..a96e15afa 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2459 +#### Version: V4.2021.07.2460 #### Bad Referrers Blocked: [7066](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/README.md b/README.md index 9fd519bdf..927840236 100755 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2021.07.2459 +#### Version: V4.2021.07.2460 #### Bad Referrers Blocked: [7066](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 6640313ce..bf41d62ed 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,7 +3,7 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2021.07.2459 +#### Version: V4.2021.07.2460 #### Bad Referrers Blocked: [7066](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index ebd22856b..431c47862 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2459 -### Updated: Mon Jul 5 13:11:20 UTC 2021 +### Version: V4.2021.07.2460 +### Updated: Mon Jul 5 13:26:47 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 901753347..6dc75839e 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2459 -### Updated: Mon Jul 5 13:11:20 UTC 2021 +### Version: V3.2021.07.2460 +### Updated: Mon Jul 5 13:26:48 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index ebd22856b..431c47862 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2459 -### Updated: Mon Jul 5 13:11:20 UTC 2021 +### Version: V4.2021.07.2460 +### Updated: Mon Jul 5 13:26:47 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 901753347..6dc75839e 100644 --- a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2459 -### Updated: Mon Jul 5 13:11:20 UTC 2021 +### Version: V3.2021.07.2460 +### Updated: Mon Jul 5 13:26:48 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 306997987..84ea522a0 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.63 1; +10.1.0.85 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 3c040781d..ec65f9e2b 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.63 0; +10.1.0.85 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 901753347..6dc75839e 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2459 -### Updated: Mon Jul 5 13:11:20 UTC 2021 +### Version: V3.2021.07.2460 +### Updated: Mon Jul 5 13:26:48 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 306997987..84ea522a0 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.63 1; +10.1.0.85 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 3c040781d..ec65f9e2b 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.63 0; +10.1.0.85 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 901753347..6dc75839e 100644 --- a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2459 -### Updated: Mon Jul 5 13:11:20 UTC 2021 +### Version: V3.2021.07.2460 +### Updated: Mon Jul 5 13:26:48 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 306997987..84ea522a0 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.63 1; +10.1.0.85 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 3c040781d..ec65f9e2b 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.63 0; +10.1.0.85 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 901753347..6dc75839e 100644 --- a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2459 -### Updated: Mon Jul 5 13:11:20 UTC 2021 +### Version: V3.2021.07.2460 +### Updated: Mon Jul 5 13:26:48 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index 4edaf066b..1db299c00 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -2459 +2460 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index 901753347..6dc75839e 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2459 -### Updated: Mon Jul 5 13:11:20 UTC 2021 +### Version: V3.2021.07.2460 +### Updated: Mon Jul 5 13:26:48 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index ebd22856b..431c47862 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2459 -### Updated: Mon Jul 5 13:11:20 UTC 2021 +### Version: V4.2021.07.2460 +### Updated: Mon Jul 5 13:26:47 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 6bf0f6f3a..310ae2ae0 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -20.94.204.162 0; +52.158.237.28 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index ebd22856b..431c47862 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2459 -### Updated: Mon Jul 5 13:11:20 UTC 2021 +### Version: V4.2021.07.2460 +### Updated: Mon Jul 5 13:26:47 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 306997987..84ea522a0 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.63 1; +10.1.0.85 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 3c040781d..ec65f9e2b 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.63 0; +10.1.0.85 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index ebd22856b..431c47862 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2459 -### Updated: Mon Jul 5 13:11:20 UTC 2021 +### Version: V4.2021.07.2460 +### Updated: Mon Jul 5 13:26:47 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 306997987..84ea522a0 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.63 1; +10.1.0.85 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 3c040781d..ec65f9e2b 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.63 0; +10.1.0.85 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index ebd22856b..431c47862 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2459 -### Updated: Mon Jul 5 13:11:20 UTC 2021 +### Version: V4.2021.07.2460 +### Updated: Mon Jul 5 13:26:47 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 306997987..84ea522a0 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.63 1; +10.1.0.85 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 3c040781d..ec65f9e2b 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.63 0; +10.1.0.85 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index ebd22856b..431c47862 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2459 -### Updated: Mon Jul 5 13:11:20 UTC 2021 +### Version: V4.2021.07.2460 +### Updated: Mon Jul 5 13:26:47 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 6bf0f6f3a..310ae2ae0 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -20.94.204.162 0; +52.158.237.28 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index d2f15d8f4..7f7cee7e8 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2459 -### Updated: Mon Jul 5 13:11:20 UTC 2021 +### Version: V4.2021.07.2460 +### Updated: Mon Jul 5 13:26:47 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 6bf0f6f3a..310ae2ae0 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -20.94.204.162 0; +52.158.237.28 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index b271e4b4f..11cfd1158 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2459 -### Updated: Mon Jul 5 13:11:20 UTC 2021 +### Version: V4.2021.07.2460 +### Updated: Mon Jul 5 13:26:47 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index 306997987..84ea522a0 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.63 1; +10.1.0.85 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 1c6f46d6e..3a5ce9da4 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -FyberSpider -Grafula -HMView -Image Fetch -Mb2345Browser -RocketCrawler -Screaming -WEBDAV -Website Quester -meanpathbot +AfD-Verbotsverfahren +Backlink-Ceck +Bolt +DomainAppender +Masscan +ProPowerBot +ScrepyBot +YoudaoBot +Zauba +Zitebot diff --git a/dev-tools/test_units/random-bots-for-test.list b/dev-tools/test_units/random-bots-for-test.list index 7caa721c5..f802b5cf3 100644 --- a/dev-tools/test_units/random-bots-for-test.list +++ b/dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ -Aboundexbot -AhrefsBot -Anarchie +360Spider +404enemy +AIBOT +ASPSeek +Abonti +AfD-Verbotsverfahren +Aipbot +Alligator Ankit -Anthill -Asterias +Apexoo +Aspiegel Attach AwarioSmartBot -BBBike -BLEXBot -BackDoorBot -BackWeb +BDCbot +BDFetch +BackStreet +Backlink-Ceck +Bandit +Barkrowler +BatchFTP +Bigfoot +Blow BlowFish -Boardreader -Buddy -BuzzSumo -CODE87 +Brandprotect +Buck +BuiltWith +Bullseye CSHttp -CazoodleBot -Cegbfeieh -CensysInspect -CherryPicker +CheTeam ChinaClaw Cocolyzebot Collector -Copyscape +CopyRightCheck Cosmos Craftbot -CrazyWebCrawler -Custo +Curious +DBLBot DSearch -DatabaseDriverMysqli -Devil -Digincore -Discoverybot +Demon Dispatch -DomCopBot -DomainCrawler -DomainStatsBot -Domains Project -Dotbot -Download Wonder -Drip -ECCP/1.0 +DittoSpyder +DomainSigmaCrawler EMail Wolf -Ecxi +EasyDL Evil Exabot -ExtractorPro -Extreme Picture Finder +Express WebPictures +Ezooms +FHscan +FemtosearchBot Fimap -FlashGet -Flunky -Fyrebot -GT::WWW -Genieo -GermCrawler -GetRight -GetWeb -Gigabot -GoZilla -GrabNet +Foobot +Freeuploader +Fuzz +Go!Zilla Grabber Grafula +GrapeFX +HEADMasterSEO +HMView HTMLparser -Haansoft +HTTrack Harvest -Hloader Humanlinks -HybridBot -IDBTE4M IRLbot Image Fetch +Indy Library +InfoTekies InterGET +Internet Ninja +InternetSeer Iria +Iskanie IstellaBot -JOC Web Spider -Jbrofuzz -JennyBot JetCar Joomla Jorgee Jyxobot -Kenjin Spider -Kinza -LNSpiderguy +Kozmosbot +LWP::Simple Larbin -Lftp -LibWeb -Libwhisker -LieBaoFast -Lightspeedsystems +LeechFTP +LeechGet +LexiBot +LinkScan Linkbot -Linkdexbot LinkextractorPro LinksManager +LinqiaRSSBot Lipperhey Spider -MFC_Tear_Sample +Lmspider +MIIxpc MJ12bot MQQBrowser -MSFrontPage MSIECrawler -MTRobot +Mag-Net Magnet -Mail.RU_Bot +Majestic SEO Majestic-SEO Majestic12 +Mass Downloader Mata Hari -MeanPath Bot -Microsoft Data Access +Mb2345Browser +Metauri +MicroMessenger +Microsoft URL Control Moblie Safari Mojolicious -Mr.4x3 -NPbot +Msrabot +Musobot Name Intelligence Navroad -NearSite Net Vampire -NetAnts +NetLyzer NetMechanic -Netcraft +NetSpider +Nettrack Netvibes -Nibbler -Niki-bot Nikto -Nuclei -Nutch -Offline Explorer +NimbleCrawler +Nmap +OPPO A33 +Offline Navigator +OnCrawl OpenLinkProfiler OpenVAS -Openfind -OrangeBot +OrangeSpider OutclicksBot -PECL::HTTP +PHPCrawl POE-Component-Client-HTTP -Page Analyzer -PageAnalyzer PageScorer +Papa Foto Petalbot +Pi-Monster Picsearch -Pimonster +PictureFinder +Piepmatz Pixray -PleaseCrawl -Psbot -Pu_iN +ProPowerBot +PxBroker PyCurl -QueryN Metasearch -RankActive -RankFlex -Rankivabot +Quick-Crawler +RSSingBot +RankActiveLinkBot RankurBot -Reaper -Recorder -RedesScrapy -SEOkicks -SEOlyticsCrawler +RealDownload +Ripper +RocketCrawler +SISTRIX SMTBot -SalesIntelligent -ScanAlert -Scrapy +ScoutJet +Screaming ScreenerBot -ScrepyBot -Searchestate +SearchmetricsBot Semrush SemrushBot Site Sucker -SiteExplorer +SiteCheckerBotCrawler SiteLockSpider -Sitebeam +SiteSnagger +SiteSucker Siteimprove +Sitevigil SlySearch Snake -Spanner +Snapbot +Snoopy +Sogou web spider Spinn3r -SputnikBot -Sqlworm -Sucuri -SuperBot +Sucker Surfbot +SurveyBot Suzuran -Swiftbot T0PHackTeam Teleport -Telesoft +TeleportPro Telesphoreo -TheNomad Thumbor +TightTwatBot Titan -Trendiction -True_Robot -TurnitinBot -Twice +Trendictionbot +TwengaBot Typhoeus -URLy Warning UnisterBot -Upflow -V-BOT VB Project +VCI Vacuum -VeriCiteCrawler -Virusdie +Vagabondo WBSearchBot WISENutbot -WPScan -WWW-Collector-E WWW-Mechanize -WWWOFFLE +WWW::Mechanize +Wallpapers/3.0 +WeSEE Web Auto -Web Collage -Web Enhancer -Web Fetch +Web Fuck +Web Pix WebAuto -WebBandit -WebCopier +WebFetch WebGo IS WebImageCollector -WebReaper -WebSauger -WebSucker -WebWhacker -WebsiteQuester -Whack -Widow -Wonderbot +WebPix +WebStripper +WebZIP +WebmasterWorldForumBot +Webshag +WebsiteExtractor +Webster +Whatweb Woobot -Wotbox -Zermelo -Zeus +Wprecon +Xaldon WebSpider +Xaldon_WebSpider +Xenu +YoudaoBot Zitebot ZmEu +ZoominfoBot +ZyBorg +arquivo-web-crawler +arquivo.pt autoemailspider +backlink-check check1.exe -coccocbot-web -crawler.feedback -demandbase-bot -eCatch -facebookscraper +crawl.sogou.com +domainsproject.org +gopher +heritrix +instabid +internetVista monitor isitwp.com -lwp-request magpie-crawler -meanpathbot -mediawords muhstik-scan oBot -page scorer -probe-image-size -scan.lol +plumanalytics +polaris version +satoristudio.net +scalaj-http +seobility seostar -sysscan +serpstatbot +sexsearcher +sitechecker.pro +sogouspider +spyfu +tAkeOut +trendiction.com +x22Mozilla zauba.io +zgrab diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 90ee84b43..3f156efaf 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,146 +1,146 @@ 01h4x.com -404checker -ASPSeek +360Spider +404enemy +AIBOT +Abonti Aboundex -Aboundexbot -Aipbot +Acunetix +AfD-Verbotsverfahren +AhrefsBot Alexibot -AllSubmitter -AlphaBot -Anarchie -Anarchy -Ankit -Attach +Alligator +Apexoo AwarioRssBot -AwarioSmartBot +BBBike BDCbot BDFetch -Backlink-Ceck -BacklinkCrawler +BackDoorBot +BackStreet +BackWeb Barkrowler -BatchFTP +Battleztar Bazinga Bitacle BlackWidow Blow -Boardreader +BlowFish Bolt Buck Buddy +BuzzSumo CCBot -CODE87 -CSHttp CazoodleBot -CensysInspect +CheTeam +CheeseBot CherryPicker +Chlooe Claritybot -Cogentbot +Cloud mapping +Collector +Copier +Copyscape Cosmos -Craftbot -Crescent -CrunchBot Curious +Custo CyotekWebCopy DBLBot -DTS Agent -DataCha0s -Devil +Demon Digincore -DigitalPebble -Disco -Discobot -Discoverybot -Dispatch +DittoSpyder +DnyzBot +DomCopBot DomainSigmaCrawler -DomainStatsBot Domains Project Dotbot -Download Wonder -EMail Siphon +Dragonfly +Ebingbong Ecxi -EroCrawler -Evil -Express WebPictures ExtLinksBot +Extreme Picture Finder +Ezooms FHscan Fimap -FrontPage +Firefox/7.0 Fuzz +FyberSpider +Fyrebot GT::WWW +Genieo +GetWeb +Getintent +Gigabot +Go!Zilla Go-Ahead-Got-It -Grafula -GrapeFX -HTMLparser +GoZilla +Gotit +HEADMasterSEO HTTrack -Haansoft -HaosouSpider Harvest -IDBTE4M -IRLbot -Iblog +Havij +Hloader Id-search +IlseBot +Image Fetch +Indy Library +InfoNaviRobot +InfoTekies +Intelliseek InterGET +Internet Ninja Iskanie IstellaBot -JamesBOT +JennyBot Jetty -Jorgee +JikeSpider +Joomla Kenjin Spider -Keyword Density +Kozmosbot LNSpiderguy -Lanshanbot -LeechGet -Lftp -Libwhisker +LeechFTP +LexiBot LieBaoFast -LinkpadBot -LinksManager +Likse +LinkScan Lipperhey Lipperhey Spider Lmspider -MJ12bot -MQQBrowser -MTRobot Mag-Net +Magnet Mail.RU_Bot -Majestic SEO Majestic-SEO -Majestic12 -Mass Downloader -MeanPath Bot +MauiBot Meanpathbot -Mediatoolkitbot MegaIndex.ru Metauri MicroMessenger Microsoft URL Control +Morfeus Fucking Scanner Mozlila -Mr.4x3 Msrabot NICErsPRO -Name Intelligence -Nessus -NetLyzer -NetZIP +NPbot +NearSite +Net Vampire +NetAnts +NetMechanic +Netcraft Netvibes -Nibbler -Niki-bot -Nikto Ninja Nuclei -Octopus +OPPO A33 Offline Explorer OnCrawl OpenLinkProfiler -Openvas -OutfoxBot +OpenVAS +OrangeBot +OrangeSpider PECL::HTTP -POE-Component-Client-HTTP -PageAnalyzer -PageScorer +PHPCrawl +Page Analyzer +PageThing.com Pandalytics -Panscient Papa Foto -PictureFinder +Petalbot +Picsearch Pixray PleaseCrawl Pu_iN @@ -148,103 +148,103 @@ Pump Quick-Crawler RSSingBot RankActive +RankActiveLinkBot +RankingBot2 +Rankivabot RankurBot +ReGet +RealDownload RebelMouse -SBIder +Recorder +RepoMonkey +RocketCrawler SEOkicks SEOkicks-Robot -SEOprofiler SEOstats +ScanAlert +Scanbot +Scrapy +Screaming ScreenerBot -Searchestate -SemrushBot -Seomoz -SiteSnagger -SiteSucker +ScrepyBot +SearchmetricsBot +Semrush +SentiBot +SeobilityBot +Siphon +SiteCheckerBotCrawler +SiteExplorer +SmartDownload Snake -Snoopy -SocialRankIOBot +Snapbot Sogou web spider +Sottopop +SpaceBison Spammen -Spinn3r -SputnikBot -Sqlmap -Sqworm +Spanner Stripper -Sucuri +SuperBot SuperHTTP Surfbot -Suzuran -Swiftbot +Szukacz +T8Abot Teleport -Telesoft -TheNomad +Telesphorep +Thumbor TightTwatBot -Titan -True_Robot -TwengaBot -Typhoeus +Trendiction +Trendictionbot +Turingos +Turnitin +Twice URLy Warning URLy.Warning -UnisterBot V-BOT -VB Project -VelenPublicWebCrawler -Virusdie -VoidEYE +VCI +Vagabondo +VidibleScraper Voltron -WEBDAV WISENutbot -WWW-Collector-E -WWW-Mechanize -WWW::Mechanize -WWWOFFLE -WallpapersHD -Web Auto -Web Fetch +Wallpapers/3.0 Web Pix -WebAuto +Web Sauger WebCollage +WebCopier WebFuck -WebImageCollector -WebReaper +WebLeacher WebWhacker WebZIP -WebmasterWorldForumBot -WebsiteQuester +Webalta Webster -Whack +Whacker +Whatweb Who.is Bot -Widow -WinHTTrack -Wonderbot Woobot Wotbox -Wprecon Xaldon_WebSpider -YoudaoBot -Zade +Xenu Zauba +Zermelo +Zeus ZmEu -ZoominfoBot -ZumBot -ZyBorg -arquivo.pt -cognitiveseo -crawler.feedback -internetVista monitor -lwp-trivial -netEstate NE Crawler -pcBrowser -polaris version -s1z.ru +backlink-check +crawler4j +demandbase-bot +eCatch +evc-batch +facebookscraper +gopher +heritrix +ips-agent +mediawords +page scorer +plumanalytics +ripz scan.lol -seostar serpstatbot -sexsearcher siteripz -sogouspider -sysscan -x09Mozilla -x22Mozilla +sp_auditbot +spyfu +voyagerx.com zauba.io +zgrab diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index d50c7ffdc..d3bd1f744 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -amigobulls.com -b2b-lounge.com -bildsuche.ru -essay-edu.biz -footballfarrago.com -gg.zzyjxs.com -hatdc.org -labafydjxa.su -pivka.xyz -site5.com +0n-line.tv +202ch.com +buybest1.biz +cheapmoncler.pw +elektrozigaretten1.postbit.com +favoritemoney.ru +gunsvicceadadebt.tk +textads.men +tizanidinehydrochloride2mg.blogspot.com +undergroundcityphoto.com diff --git a/dev-tools/test_units/random-referrers-for-test.list b/dev-tools/test_units/random-referrers-for-test.list index 2b1fb20a5..7ca6b0091 100644 --- a/dev-tools/test_units/random-referrers-for-test.list +++ b/dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -03e.info -101flag.ru -10bet.com -178evakuator178.ru -18ps.ru -1adult.com -1kinobig.ru -21h2o.com -2345.com -24x7-server-support.site -2itech.ru -34.gs -3weekdiet.com -51unlim.ru -5kstore.com -6128786.com -7makemoneyonline.com -991mostfm.co.id -aaa38852e886ac4af1a3cff9b47cab6272.com -acheterviagrafr24.com -acmebtn.ml -acortarurl.es -actionnooz.com -acunetix-referrer.com -adclickservice.com -adloads.com -alarmobninsk.ru -albamargroup.com -alborzan.com +123cha.com +1millionusd.xyz +202ch.com +20pascals.nl +273-fz.ru +3xforum.ro +4kepics.com +4pp13.com +9med.net +aa9d046aab36af4ff182f097f840430d51.com +absolute-s.ru +acads.net +actulite.com +adclickthru.net +adconscious.com +adhome.biz +adultwebhosting.info +affiliate-fr.com +agecheckadult.com +ald2014.org alekseevec.ru -algarveglobal.com -algerianembassy.co.in -alif-ba-ta.com -allsilver925.co.il -alphavisions.net -amateurlivechat.org -amung.us -angigreene.com -apiadanaknet-a.akamaihd.net -apibatbrowsecom-a.akamaihd.net -apiholdingmypage-a.akamaihd.net -apioasisspacenet-a.akamaihd.net -apispringsmartne-a.akamaihd.net -apple.com-cleaner.systems +alessandraleone.com +alf-img.com +alienwheels.de +alt-servis.ru +amazingpic.net +ampower.me +analnoeporno.tv +animal36.com +apiqualitinknet-a.akamaihd.net +app-ready.xyz +appartement-stumm.at +aproposde.com arcteryxsale.online +arcteryxstore.online +arendovalka.xyz +arius.tech +artdestshop.eu artparquet.ru -asmxsatadriverin.aircus.com -asrv-a.akamoihd.net +asiavirtualsolutions.com +asrvvv-a.akamaihd.net atlant-auto.info audiofree.ru -automatic-seo.com -avtointeres.ru -azartniy-bonus.com -azbaseimages.net -azinoofficial777.ru -backiee.com -balois.worldbreak.com -baoxaydung.com.vn -barnfurnituremart.com -battleforupdating.site -bbtec.net -beclean-nn.ru -big-cash.net -binomo.kz -bird1.ru -bitnote.co -blavia.00author.com -blockworld.ru -blogporn.in -bluerobot.info -bonux.nextview.ru -bonvillan.com +avkzarabotok.info +ayodhya.co +azbuka-mo.ru +backuperwebcam.weebly.com +baditri.com +bangla-chat-uk-paralud.ga +bankcrediti.ru +bannerbank.ru +bensbargains.net +bestdraws.com +bestmobilityscooterstoday.com +bestpriceninja.com +bestserials.com +bestssaker.com +betteroffers.review +bidr.trellian.com +bimatoprost-careprost.com.ua +bioinnovate.co +bitcoinpile.com +bizzliving.com +bloglag.com +bolezniorganov.ru +bongacams.com +bonzbuddy.com bookmark4you.com -bosefux.esy.es -braip.com.br -bristolhotel.com.ua -brothers-smaller.ru -bugof.gq -buigas.00it.com -bupropion-sr-150-mg.us -businesxxl.com -buy-forum.ru -buyk.host.sk -bvlgariring.xyz -callcafe.info -cand.jp -carstrends2015.com -cartierbraceletsreplica.pw +boole.onlinewebshop.net +brandehk.dk +brus-vsem.ru +burn-fat.ga +buscarfoto.com +buytizanidineonlinenoprescription.blogspot.com +buzzurl.jp +cakemediahq.com.au +cakesplus.com.au +candy-glam-hp.com +cannibalcheerleader.com +carapics.com +cardul.ru +carloans.com cartierreplica.xyz -cartiertrinity.xyz -casinorewards.info -ccjp.eu -cctva.tv -cheatcode-lita12.rhcloud.com +cartierring.xyz +cartierwatch.xyz +casablancamanor.co.za +cayado.snn.gr +cdncash.net +cdncash.org +celejihad.info +cellfun.mobi +chatroulette.world checkhit.com -chuckguilford.com -cialischstgerts.com -clickbanksites.info -clickcash.com -clipartnew.com -clothesforcash.com -collegerentals.com -com-supportcenter.website -communisave.co.za -connectingsingles.com -converse.ddsoldes.fr -cookie-law-enforcement-aa.xyz -copypaste.traderzplanet.in -counter.bloke.com -culturevie.info -custodieva.ru -customsua.com.ua -dailyfinancefix.com -dailystrength.org -daisye.top -dalavia.ru -dame-ns.kz -daymusam.com -depositfiles-porn.ga -derevesendeco.com -devochki.top -dlya-android.org -documentsite.net -dogbreedspicture.net -dokfilms.net -domaineaneblanc.com -dorratex.tn -dptaughtme.com -drillsaw.ru +chinacheapelitejerseys.com +chlooe.com +choosecuisine.com +clean-start.net +cleanallvirus.com +click2pawn.com +clknsee.com +club-samodelkin.ru +coderstate.com +coldfilm.ru +comeondog.info +compliance-jane.top +conocer-sanabria.com +cornomase.win +craftinsta.ru +cubook.supernew.org +custom-electric-guitar.com +cxpromote.com +cyprusbuyproperties.com +dairyindia.in +damasarenai.info +davidovic.info +dayibiao.com +dbmkfhqk.bloger.index.hr +de.zapmeta.com +deda-moroza-zakaz.ru +detsky-nabytek.info +dieswaene.com +diffbot.com +digital-sale.su +discountliv.com +disruptingdinnerparties.com +div.as +djekxa.ru +dojki365.online +domcran.net +donvito.unas.cz e-commerce-seo.com -eachdayisagift.review -eas-seo.com -elektrischezigaretten1.blogse.nl -elektronischezigarettekaufen2.tumblr.com -elvenar.com -elvenmachine.com -energy-ua.com -eupornstar.info -evaashop.ru -evehemming.blogspot.com.au -excitacion.info -expdom.com -eyelike.com.ua -ezigarettekaufen2.blox.pl -ezigarettenkaufen2.dreamwidth.org -fantasticpixcool.com -fbdownloader.com -ferieboligkbh.dk -ferretsoft.com -fetlifeblog.com -figuringmoneyout.com -finansov.info -findclan.org -findthe.pet -flash4fun.com -footbalive.org +eaplay.ru +ecommerce-seo.com +educontest.net +ee77ee.com +eets.net +efkt.jp +ekspresihati.info +eldiariodeguadalajara.com +elektrozigarettekaufen2.blogse.nl +elitedollars.com +embedle.com +englate.com +enthuse.computernetworksonline.com +erotikstories.ru +espaceinventoristes.com +eu-cookie-law.info +europages.com.ru +ez8motelseaworldsandiego.com +ezigarettenkaufen1.pagina.gr +ezigarettenshop2.postbit.com +factorynightclub.com +famix.xyz +fanoboi.com +favornews.com +fighrofacciufreesig.ga +filmgo.ru +find1friend.com +findacheaplawyers.com +fiverr.com +flavors.me +flex4launch.ru forex.osobye.ru -fortwosmartcar.pw -forum69.info -foxjuegos.com -foxydeal.com -frameimage.org -free-traffic.xyz -freevpn.space -freza-sverlo.ru -fuck-paid-share-buttons.xyz -fullgirl.ru -funponsel.com -fxund.us -fyxabomiw.ru -gambarxkata.co -game300.ru -gamerextra.com -gamingspark.com -gate5.co.za -gateway.zscalertwo.net -gdzkurokam.ru -germes-trans.com -getaclueamerica.com -gilsonchiro.xyz +formseo.com +forzeronly.com +foto-basa.com +free-laptop-reward.com +frighteningremain.cf +front.to +fyl.com.ru +gameonasia.com +gamesprite.me +gayxperience.com +ge0ip.net +geckoandfly.com +gentamicineyedrops.blogspot.com +getlamborghini.ga +gigapeta.com girlspicsa.com -gk170.ru -glastecfilms.com.my +global-ics.co.za go2album.com go2mike.ru -gok-kasten.net -goldenggames.com -googlemare.com -googst2.ru -gotomontenegro.net -gtopstats.com -guruofcasino.com -helicalpile.us -hermesreplica.pw +goforexvps.com +greendream.com.ua +gribkovye-zabolevaniya.com +gsasearchengineranker.pw +hanink.biz.ly hermesreplica.win -heroz.fr -hikesearch.net -histats.com -hit-men.men -homeinns.com -homelygarden.com -hoste.octopis.com -hosting-tracker.com -hotchatdate.com -houseofrose.com +hostingclub.lk +hotblog.top +hotdl.in +hotgirlhdwallpaper.com +hotkeys.com +hotxnights.info +howtoclean.club hoztorg-opt.ru -hscsscotland.com -hully.altervista.org -hyiphunter.org -i-hobot.ru -igtools.club -ilte.info -imagecoolpub.com -imk.com.ua -immobiliaremassaro.com -impisr.ru -infektsii.com +humbmosquina.tk +hundejo.com +hustoon.over-blog.com +iboss.com +ilovevitaly.com infobanks.ru -intim-uslugi.info -intimshop-fantasy.ru -iphantom.com -iptool.xyz -isotoner.com -ix20.ru -ixora.pro -izismile.com -j33x.com -jaxcube.info -jerseychinabizwholesale.com -jovencitas.gratis -jrcigars.com -justucalling32211123456789.tk -kakadu-interior.com.ua -kartiniresto.com -kashubadesign.ru -keywordsuggest.org -kfon.eu -kinobaks.com -kinoplen.ru -kol-energo.ru -konturkrasoty.ru -kostenlos-sexvideos.com -kumuk.info -kupit-adenu.ru +instabid.tech +insurple.com +internetproviderstucson.com +investmac.com +jabimgo.pw +jackpotchances.com +javatex.co.id +joingames.org +jolic2.com +jurajskie.info +kaidalibor.de +kalb.ru +kamagragelusa.net +kambasoft.com +kamen-e.ru +kareliatobacco.ru +kazinogames.lv +kerch.site +kinky-fetishes.com +kinobest.pl +kongoultry.net +kotaku.com +kovrenok.ru +krasnodar.xrus.org kurbappeal.info -kw21.org -landreferat.ru -lapitec.eu -larutti.ru -lawyersinfo.org -li-er.ru -likesuccess.com -liky.co.ua -limads.men -liveinternet.ro -lmrauction.com -loadopia.com -lombia.com -lost-alpha.ru -m4ever.net -managerpak204.weebly.com -manuscript.su -matrixalchemy.com -matsdale.com -mature.free-websites.com -me-ke.com +kvartir-remont.biz +lacave.ntic.fr +lider-zhaluzi.kiev.ua +likrot.com +liveinternet.ru +liveshoppersmac.com +localflirtbuddies.com +localmatchbook.com +lomza.info +longgreen.info +louisvuittonoutletstore.net +lulea-auktionsverk.se +m3gadownload.pl +mainhp.net +makenahartlin.com +marmitaco.cat meble-bogart.info -memberty.com -mensandals.xyz -mere.host.sk -metabar.ru -mhtr.be -microsoftportal.net -mielec.pl -misandesign.se -mistr-x.org -mobifunapp.weebly.com -monclercheap.xyz -moroccosurfadventures.com -morpicert.pw -mriyadh.com -msfsaar.de -mtmtv.info -mundoaberrante.com -musicspire.online -muycerdas.xxx -myhydros.org -myiptest.com -mylida.org -mypets.by -mysex21.com -nationalbreakdown.com -net-profits.xyz -netpics.org -newstudio.tv -nfljerseyscheapchinabiz.com -nicovideo.jp -nippon-bearings.ru -nodding-passion.tk -normalegal.ru -nottyu.xyz -obsessionphrases.com -ok.ru +medanestesia.ru +medizinreisen.de +metaxalonevstizanidine.blogspot.com +milblueprint.com +mini.7zap.com +mirtorrent.net +mnogabukaff.net +mnogolok.info +mobile.ok.ru +mobioffertrck.com +moetomnenie.com +moivestiy.biz +mole.pluto.ro +money-every-day.com +montazhnic.ru +mossmesi.com +motherboard.vice.com +moviemail-online.co.uk +mrcsa.com.au +mrwhite.biz +muizre.ru +musezone.ru +musezone.su +mybinaryoptionsrobot.com +mycaf.it +na15.ru +nacap.ru +nailsimg.com +napalm51.nut.cc +nash-krym.info +nodup.ru +noumeda.com +oballergiya.ru +offside2.5v.pl +olympescort.com omoikiri-japan.ru -onlinebay.ru -onlineslotmaschine.com -onlinewritingjobs17.blogspot.ru -outclicks.net -page2rss.com -partner-pop.men -partybunny.ru -pcboa.se -penisvergrotendepillennl.ovh -pensplan4u.com -perfectpracticeweb.com -perso.wanadoo.es -petsblogroll.com -peugeot-club.org -phobia.us -phrcialiled.com -pillscheap24h.com +one-gear.com +onemactrckr.com +onlinemegax.com +onlineporno.site +ontargetseo.us +ooo-gotovie.ru +optikremont.ru +ordernorxx.com +orel-reshka.net +painting-planet.com +palvira.com.ua +panchro.co.uk +patterntrader-en.com +pensplan.com +pepperstyle.ru +pewit.pw +pic2fly.com pinup-casino1.ru -pixshark.com -pizdopletka.club -pizza-tycoon.com -play.leadzupc.com -pliks.pl -polcin.de -poligon.com -popunder.net -pornmania.pl -porno-play.net -pornofiljmi.com +pizdeishn.com +pl.id-forex.com +pons-presse.com +porngalleries.top +pornokorol.com pornosee.info -pornosmola.info -pornstartits.xblog.in -pornzone.tv -pretty-mart.com -prodess.ru -professionaldieselcare.com -prombudpostach.com.ua +portnoff.od.ua +pp-budpostach.com.ua +predmety.in.ua +presleycollectibles.com +profitsport.club +promodj.com promover.org -prostitutki-kharkova.org -prostitutki-kiev.org -rada.ru -rankia.com -razleton.com -redirlock.com +psvita.ru +pussyspace.net +ranksonic.com +rating-casino2021.ru +razorweb-a.akamaihd.net +realtytimes.com +rebelmouse.com rednise.com -registrydomainservices.com -remont-ustanovka-tehniki.ru -remorcicomerciale.ro -remybutler.fr -resellerclub.com -rethinkwasteni.info -reward-survey.net -rfserial.net -riralmolamsaca.tk -robot-forex.biz -rollercoin.com -royalcar-ufa.ru -rozalli.com -rvzr-a.akamaihd.net -sakhboard.ru +referencemoi.com +refudiatethissarah.info +relax.ru +remontvsamare.su +responsinator.com +retreatia.com +rewardpoll.com +rialp.getenjoyment.net +rome2rio.com +roznica.com.ua +rus-pornuha.com +saltspray.ru salutmontreal.com -sandhillsonline.com -saratov.xrus.org -sarm.tk +santechnik.jimdo.com +sarf3omlat.com saugatuck.com -sawin.beth.webd.pl -scanner-mark.top -score-ads.men -searchmywindow-a.akamaihd.net -seksvideoonlain.com -seorankinglinks.com -servicecenter.co.ua -sex-dating.co -sexblog.pw +saveindex.xyz +savingsslider-a.akamaihd.net +sbt-aqua.ru +scanner-alexa.top +scanner-ivan.top +scanner-margo.top +scrapy.org +search-error.com +secretscook.ru +seeresultshub-a.akamaihd.net +seksotur.ru +seobility.net +sexgalleries.top sexphoto.site -sexpornotales.net -sextracker.be -sexvideo-sex.com -sexystrippe.info -sexyteens.hol.es -sharebutton.net -sheerseo.com -shemalegalls.blogporn.in -shinikiev.com.ua -shisha-swag.de -shohanb.com -shymkent.xkaz.org -sibdevice.ru -sibecoprom.ru -simon3.ru -simple-share-buttons.com -simplesite.com -sims-sims.ru -siteexpress.co.il -snip.tw -soc-econom-problems.ru -social-button.xyz +sexspornotub.com +shillyourcoins.com +shivafurnishings.com +shmetall.com.ua +shop-electron.ru +sinel.info +skincrate.net +sledstvie-veli.net +slkrm.ru +smallseotools.com +soft-program.com +softtor.com +softwaretrend.net softxaker.ru -soheavyblog.com -sophang8.com -spl63.fr -sports-supplements.us -stroilka.info -stroiminsk.org -studentguide.ru -studiofmp.com -stuff-about-money.com -suggest-keywords.com +sohoindia.net +sonnikforme.ru +spb-plitka.ru +spb.afora.ru +sprttrack.com +spy-app.info +sr-rekneskap.no +stackthatbucks.com +stal-rulon.ru +standardchartered-forex.com +stariy-baku.com +startwp.org +statoutlook.info +steame.ru +steebook.com +stroydetali.ru +sukarame.net +sumo.com +superinterstitial.com supernew.org -svarkagid.com -swinger-mobil.net -tastyfoodideas.com -taximytishi.ru -technika-remont.ru -tecnoteakviareggio.it -teenfuck.tv -thecoolimages.net -thetoiletpaper.com -tizanidineduringpregnancy.blogspot.com -tizanidinehcl2mgsideeffects.blogspot.com -tizanidinehcl4mgisitanarcotic.blogspot.com +swagbucks.com +syvertsen-da.no +tds-advert002.info +telegramdownload10.com +teletype.in +thebluenoodle.com +thecoral.com.br +themestotal.com +titan-cloud.life +tivolibasket.it +tizanidineandgabapentin.blogspot.com tizanidineinteractions.blogspot.com -tizanidineusedfor.blogspot.com -tk-assortiment.ru -tmearegion26.com -top-karkas.ru -top1-seo-service.com +tizanidinevssoma.blogspot.com +tizanidinewithdrawalsymptoms.blogspot.com +tkanorganizma.ru +tn811.us +tnctrx.com +tobeyouday.win +toonfamilies.net top10-online-games.com -tovaroboom.vast.ru -tramadolandtizanidine.blogspot.com -tx41tclega.ru -uchetunet.su -ultimateclassicrock.com +topdownloads.ru +topvidos.ru +torrent-to-magnet.com +torrentdownloadhub.com +tourismvictoria.com +toyota.7zap.com +track2.shop +trackmedia101.com +traffic100.com +trafficreceiver.club +trafficstars.com +traffixer.com +trainoffend.ml +truck-rece.life +tubo360.com +twiclub.in +twinderbella.com +twittrading.com +twojebook.pl +tyumen.xrus.org +u555u.info +ua-company.ru +ultramart.biz +umekana.ru unitygame3d.com -upupa.net -urlopener.blogspot.com.au -uzporno.mobi -velobikestock.com -vikistars.com -virus-schutzmasken.de -visitcambridge.org -viveresaniesnelli.it -vmnmvzsmn.over-blog.com -voluumtracker1.com -vremya.eu -vroze.com -vvpg.ru -vzlomsn.org -vzlomtw.com -walkme.com -waterefficiency.co +unrealcommander.com +urlcut.ru +uvozdeckych.info +vacuumscleaner.com +validdomain.xyz +varikoz24.com +ventelnos.com +via-energy-order.com +viagra.pp.ua +video-chat.love +videositename.com +videtubs.pl +vinylvault.co.uk +vipromoffers.com +virtualbb.com +visa-china.ru +viteonlusarezzo.it +vkontaktemusic.ru +volume-pills.biz +w7s.ru wcb.su -webcamdevochka.com -webextract.profound.net -webix.me -welcomeauto.ru -westsextube.com -white-truck.life -whiteelephantwellington.com -wholesalecheapjerseysfree.com -wholesalejerseychinaoutlet.com +website-analyzer.info +websiteaccountant.de +websitevaluebot.com +weclipart.com +while.cheapwebsitehoster.com +whiteproduct.com wildworld.site -wineitudes.wordpress.com -wineration.com +winterclassichockeyjerseys.com wnhjavlhezp.gq -woman-orgasm.ru -word-vorlagen.xyz -worldoffiles.ru -www.888.com -x69ty.ru -xlovecam.com -xn-----6kcaabbafhu7cskl7akvongwpo7hvjj.xn--p1ai -xn--80ajbshivpvn2i.xn--p1ai -xn--b1adccaf1bzj.xn--p1ai -xpresscare.ru -xxxsiterips.xyz -youbrainboost.asia -yougotanewdomain.com -youradexchange.com -yourmovies.pl -yourserverisdown.com -ypmuseum.ru -zagadki.in.ua -zaloadi.ru -zaloro.com -zeg-distribution.com -zeroredirect2.com -zeroredirect7.com -zhuravlev.info -zigarettenonlinekaufen2.bloog.pl -zionstar.net -zirondelli.it -znaturaloriginal.com -zona-aqua.ru -zoogdinsney.com -zootravel.com +wrz0iuebwhp5fg.freeddns.com +wurr.voila.net +www.souvenirua.com +wyniki-lista.pl +x-diesel.info +xclicks.net +xdoza.com +xedserver.com +xingzi-vision.com +xmnb.net +xn-----6kcaacnblni5c5bicdpcmficy.xn--p1ai +xn----8sbdbjgb1ap7a9c4czbh.xn--p1acf +xn----itbeirbjbi7bc6bh2d.xn--p1ai +xn--80aafb2a.xn--p1ai +xn--80aagddcgkbcqbad7amllnejg6dya.xn--p1ai +xn--lifehacer-1rb.com +xnxxandxvideos.com +xolodremont.ru +xtrafficplus.com +xvideospornoru.com +xxxguitars.com +yaoguangdj.com +yatrk.xyz +yebocasino.com +yellowstonesafaritours.com +yoluxuryevents.com +yottos.com +yougame.biz +yourothersite.com +yoursearch.me +yoursite.com +yubikk.info +yur-p.ru +z-master.ru +zaimite.ru +zakazvzloma.com +zeroredirect.com +zetmaster.ru +zlatnajesen.com +zoogdisany.com +zscalertwo.net +zvezdagedon.ru diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index 1e96e70fb..c24618c95 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -100searchengines.com -178evakuator178.ru -4rent.ru -4ureyesonly.com -5forex.ru -6hopping.com -9icmzvn6.website -aa81bf391151884adfa3dd677e41f94be1.com -aa9bd78f328a6a41279d0fad0a88df1901.com -aadroid.net -acmebtn.ml -adinterax.com -adtech.fr -albuteroli.com -algarveglobal.com -allkrim.com -amt-k.ru -approvedlocal.co.za -arendas.net -asscenihotosidea.blogspot.co.za -atmagroup.ru -atmovs.com -autochoixspinelli.com +123locker.com +2020iscoming.info +202ch.com +20pascals.nl +2itech.ru +300richmond.co.nz +8lufu.com +999webdesign.com +abcdeg.xyz +abiente.ru +adloads.net +admanaerofoil.com +ads-seo.men +agardomains.com +agysya.ru +akama.com +anjalika.co.in +apccargo.com +apiadanaknet-a.akamaihd.net +apiholdingmypage-a.akamaihd.net +arewater.com +arvut.org automatic-seo.com -autoua.top -azartmix.com -babespcs.com -baikaleminer.com -beachtoday.ru -beclean-nn.ru -berdasovivan.ru -bestssaker.com -betune.onlinewebshop.net -binomo.kz -bitcoinpile.com -bjanshee.ru -brainboosting.club +aviav.ru +bannerspace.com +bestbrainprod.win +bestcoin.cc +bestprofits-there.com +betteroffers.review +biketank.ga +blog.yam.com +bobba.dzaba.com +bongacams.com +boom-celebs.com brothers-smaller.ru -bum.com.ru -businescoop.men -buy-meds24.com -buyantiviralwp.com -callawaygolfoutlet.online -capacitacionyaprendizaje.com -capsderfudd.tk -cartierloveringreplica.xyz -cdncash.net -centrdebut.ru -chatroulette.si +bugshoot.cn +canvas.gloverid.site +carsdined.org +cartierlove.xyz +cartierlovestore.xyz +cdncash.com cheap-pills-norx.com -cheapestjerseys-wholesale.com -chocoslim.pro -cialiswithoutadoctor.net -citetick.com -cleanmypc.com -clicksor.net -cocolyze.com -com-cleaner.systems -compliance-fred.top -concordexoticrentals.com -contentlook.co -contentsexpress.com -cowblog.fr -d-black.bz -d0t.ru -dailystrength.org -danceuniverse.ru -dieswaene.com -dimkino.ru -doreenblog.online -dostavka-v-ukrainu.ru -downloadeer.net -dyshagi.ru -eas-seo.com -education-cz.ru -ege-essay.ru -elvel.com.ua -emergencyneeds.org -eonpal.com +cheapjerseysa.com +cialis-samples.com +cinemaflix.website +clash-clans.ru +click2pawn.com +coldfilm.ru +com-onlinesupport.site +compliance-checker.info +conciergegroup.org +confib.ifmo.ru +constantaservice.net +coslab.club +countercrazy.com +covid-schutzmasken.de +crazyprotocol.com +cubook.supernew.org +degerlund.net +delta-line.men +dinkolove.ya.ru +divisioncore.com +doctissimo.fr +doktoronline.no +dollartree.info +downloader12.ru +dyt.net +e-kwiaciarz.pl +eaplay.ru +elitepcgames.com +elkacentr.ru +elstal.com.pl +eng-lyrics.com +enge-fotzen.info escortslet.net -estelight.ru -fastcrawl.com -fenoyl.batcave.net -fetlifeblog.com -filesmonster.porn -fireads.men -fix-website-errors.com -forex.osobye.ru -forum.doctissimo.fr -free-deals.faith -fungamelands.com -generalporn.org -goldenggames.com -gonextmedia.com -groupmoney.ru -guidefs.ru -gungamesz.com -heartofpayne.xyz -hexpilot.com -hoholikik.club -hop.clickbank.net -houdom.net -humanorightswatch.org -ibmdatamanagement.co -imagefinder.site -iminent.com -imitex-plus.ru -inboxdollars.com -indo-export.ru -insurple.com +eshop4u.jp +este-line.com.ua +evening-dating-club.info +everypony.ru +executehosting.com +f5mtrack.com +fefo.gdn +flex4launch.ru +flooringinstallation-edmonton.com +footballfarrago.com +foto-sisek.porngalleries.top +freemags.cc +freeuploader.ml +fuck-paid-share-buttons.xyz +galaxyflowers.ru +gembird.com +goldandcard.ru +goodnightjournal.com +googlepositions.com +greatgrace.ru +greenidesign.co +handicapvantoday.com +hentaiheroes.com +hetmanship.xyz +historichometeam.com +housediz.com +i-service.kz +id-forex.com +iflycapetown.co.za +ilovevitaly.ru +imperiafilm.ru +infobabki.ru +integritylandscapeservices.com +intellego.info +invitefashion.com ipostroika.ru -itch.io -itrevolution.cf -itsdp3.com -iwanttodeliver.com -izhevsk.xrus.org -jerseychinabizwholesale.us -kalandranis.gr -kamagragelusa.net -kinogolos.ru -konica.kz -konturkrasoty.ru -kurbappeal.info -louboutinshoes.xyz -luckyday.world -m.b00kmarks.com -magicdiet.gq -mailemedicinals.com -marcoislandvacations.net -marketingtechniques.info -mediafresh.online -medicines-choice.com -megaindex.ru -members.ghanaweb.com -mmstat.com -moivestiy.biz -monarhs.info -moscow.online-podarki.com -movpod.in -multgo.ru +islandminingsupply.wordpress.com +jav-fetish.com +jestr.org +johnnyhaley.top +journalhome.com +jpcycles.com +kazka.ru +kerwinandcariza.com +keywordbasket.com +khafre.us +kinogonew.ru +kinostorm.net +klikbonus.com +kolotiloff.ru +krasivye-devushki.net +lafourmiliaire.com +lalalove.ru +laparfumotec.com +law-check-eight.xyz +laxdrills.com +ldrtrack.com +lenvred.org +life.biz.ua +lifebyleese.com +lion.conboy.us +lombardfinder.ru +madot.onlinewebshop.net +meddesk.ru +megahdporno.net +meget.co.za +mirzonru.net +misandesign.se +moncleroutletonline.pw +moncleroutletonline.win +moscow-region.ru +mountainstream.ms +nac-bearings.ru narkologiya-peterburg.ru -nightvision746.weebly.com -nisuturnetdgu.tk -nofreezingmac.work -o-o-11-o-o.com -oballergiya.ru -okout.ru -online-x.ru -onlinemeetingnow.com -onstrapon.purplesphere.in -openfrost.net +nastydollars.com +naval.jislaaik.com +newtechspb.ru +nomuos.it +notaria-desalas.com +notasprensa.info +ochistka-stokov.ru +offers.bycontext.com +office-windows.ru +okeinfo.online +online-templatestore.com +openmediasoft.com osb.se11.ru -partybunny.ru -pekori.to -piluli.info -pinwallpaper.xyz -playmsn.com -plusnetwork.com -popunder.ru -pornoelita.info -pornoforadult.com -pozdrawleniya.com -prchecker.info -priora-2.com -privatov-zapisi.ru -prosperent.com -prostitutki-kharkova.org -psoriasis-file.trade -ptr.ruvds.com -pulseonclick.com -rating-bestcasino.com -razleton.com -razyboard.com -realmonte.net -realnye-otzyvy.info -redirlock.com -reining.lovasszovetseg.hu -remont-fridge-tv.ru -russkoe-zdorovie.ru -samoiedo.it -sbricur.com -sei80.com -seksvideoonlain.com -seminarygeorgia59.ga -seo-tools-optimizing.com -seofirmreviewsus.info -sexytrend.ru -shareyards.com -shodanhq.com -shoppingmiracles.co.uk -si-unique.com -sibtest.ru -sibvitr.ru -site.ru -slkrm.ru -smartbalanceworld.com -sobecjvuwa.com.ru -socialbuttons.xyz -steame.ru -studiokamyk.com.pl -styro.ru -suggest-keywords.com -tam-gde-more.ru -tastyfoodideas.com -telefonsexi.com -theautoprofit.ml -ticketsys.inetwd.com -tizanidine4mgstreetvalue.blogspot.com -tizanidineandgabapentin.blogspot.com -tizanidinemusclerelaxant.blogspot.com -tnctrx.com -topflownews.com -torrent-to-magnet.com -totu.info -trubywriting.com -twiclub.in -uac.net.au -udsgame.online -ukrobstep.com -ultimatesetnewfreeallsoftupgradesystems.pw -uni.me -uroki.net -uzungil.com -vancleefreplica.pw -vapsy.com -vekzdorov.ru -velpanex.ru -viteonlusarezzo.it -wait3sec.org -warezkeeper.com -watch-movies.ru -wemarketing.se -winner7777.net -wma-x.com -wyniki-lista.pl -xn----7sbifcamovvfggw9d.xn--p1ai -xn--80abgj3a5acid6ghs.top -xvideosporn.biz -xz618.com -yoshkarola.zrus.org -youtubedownload.org -z-master.ru -za-fun-offer.com -zaloadi.ru -zaloro.com -zarabotat-na-sajte.ru -zastroyka.org -zazagames.org -zed21.net -zeroredirect2.com +oz-offers.com +ozoz.it +palocco.it +parfusale.se +partnersafe.men +partnerworkroom.men +pattersonsweb.com +pistonclasico.com +pixgood.com +play.leadzupc.com +poem-paying.gq +poppen-nw.net +pornobrazzers.biz +pornosok.ru +posible.net +pregnant.guru +princeadvantagesales.com +privacyassistant.net +raisedseo.com +razorweb-a.akamaihd.net +registry-cleaner.net +rocketchange.ru +rossanasaavedra.net +rumamba.com +runstocks.com +sa-rewards.co.za +sad-torg.com.ua +scanner-jane.top +scanner-nelson.top +scripted.com +seeresultshub-a.akamaihd.net +seo18.su +sextracker.com +siteaero.com +siteonomy.com +sivs.ru +socialmonkee.com +softxaker.ru +sogimlecal.tk +sortthemesitesby.com +sports-supplements.us +strfls.com +t3chtonic.com +technopellet.gr +tengohydar.tk +the-trader.net +thepokertimer.com +tizanidinevscyclobenzaprine.blogspot.com +tomck.com +tracklead.net +truebeauty.cc +tsyndicate.com +twittrading.com +tzritel.tk +uamtrk.com +ucsol.ru +unimodemhalfduplefw.pen.io +uptimebot.net +vchulkah.net +veerotech.com +veles.shop +via-energy-acquistare.com +videtubs.pl +viktoria-center.ru +vip-parfumeria.ru +visionwell.com.cn +visitcambridge.org +vzlomfb.com +webextract.profound.net +webjam.com +website-speed-check.site +wejdz-tu.pl +wemedinc.com +wildnatureimages.com +xev.ru +xfluro.com +xn----8sbhefaln6acifdaon5c6f4axh.xn--p1ai +xnxx-n.com +yebocasino.co.za +yoopsie.com +youjizz.vc +youporn-ru.com +yoursite.com +zapiszto.pl +zebradudka.com +zeleznobeton.ru +zetmaster.ru +zona-aqua.ru diff --git a/dev-tools/test_units/ratelimittest-master.txt b/dev-tools/test_units/ratelimittest-master.txt index 3656720f2..f0474a154 100644 --- a/dev-tools/test_units/ratelimittest-master.txt +++ b/dev-tools/test_units/ratelimittest-master.txt @@ -581,14 +581,4 @@

Welcome to the NginX Ultimate Bad Bot Blocker

This is merely used for testing !!!

- - - -Welcome to NginX - - -
-

Welcome to the NginX Ultimate Bad Bot Blocker

-

This is merely used for testing !!!

- \ No newline at end of file diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index 3c040781d..ec65f9e2b 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.63 0; +10.1.0.85 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index bdd8c12bb..e365759d7 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -11230,3 +11230,1251 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2460 +### Updated: Mon Jul 5 13:26:49 UTC 2021 +### Bad Bot Count: 615 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: OPPO A33 +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ + diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index bdd8c12bb..e365759d7 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -11230,3 +11230,1251 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2460 +### Updated: Mon Jul 5 13:26:49 UTC 2021 +### Bad Bot Count: 615 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: OPPO A33 +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ +