From 2c40675958a45eb672252713d8c77e093fdaba94 Mon Sep 17 00:00:00 2001 From: mitchellkrogza Date: Fri, 9 Jul 2021 08:49:08 +0000 Subject: [PATCH] V4.2021.07.2464 [ci skip] --- AUTO-CONFIGURATION.md | 2 +- MANUAL-CONFIGURATION.md | 2 +- README.md | 2 +- _google_analytics_ghost_spam/README.md | 2 +- .../etc/nginx/conf.d/globalblacklist.conf | 4 +- conf.d/globalblacklist-testing-version.conf | 4 +- conf.d/globalblacklist.conf | 4 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- dev-tools/buildnumber | 2 +- dev-tools/globalblacklist-testing.template | 4 +- dev-tools/globalblacklist.template | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- dev-tools/test_units/blacklist-ips.conf | 2 +- .../random-bots-for-test-quick.list | 18 +- .../test_units/random-bots-for-test.list | 310 ++-- .../random-bots-for-whitelist-test.list | 292 ++-- .../random-referrers-for-test-quick.list | 20 +- .../test_units/random-referrers-for-test.list | 944 ++++++------- .../random-referrers-for-whitelist-test.list | 478 +++---- dev-tools/test_units/whitelist-ips.conf | 2 +- dev-tools/tmp/robots.txt | 1248 +++++++++++++++++ robots.txt/robots.txt | 1248 +++++++++++++++++ 45 files changed, 3579 insertions(+), 1083 deletions(-) diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index c1a477fe4..b6583683d 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2463 +#### Version: V4.2021.07.2464 #### Bad Referrers Blocked: [7067](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index c338c35a1..a030a180f 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2463 +#### Version: V4.2021.07.2464 #### Bad Referrers Blocked: [7067](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/README.md b/README.md index 5876e8f35..8e1ef9f7a 100755 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2021.07.2463 +#### Version: V4.2021.07.2464 #### Bad Referrers Blocked: [7067](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 22d05b950..587b30f8b 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,7 +3,7 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2021.07.2463 +#### Version: V4.2021.07.2464 #### Bad Referrers Blocked: [7067](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 5149f32dd..4e92abccf 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2463 -### Updated: Fri Jul 9 07:07:55 UTC 2021 +### Version: V4.2021.07.2464 +### Updated: Fri Jul 9 08:43:17 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index b145a26e7..4d1c00c0f 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2463 -### Updated: Fri Jul 9 07:07:56 UTC 2021 +### Version: V3.2021.07.2464 +### Updated: Fri Jul 9 08:43:17 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 5149f32dd..4e92abccf 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2463 -### Updated: Fri Jul 9 07:07:55 UTC 2021 +### Version: V4.2021.07.2464 +### Updated: Fri Jul 9 08:43:17 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf index b145a26e7..4d1c00c0f 100644 --- a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2463 -### Updated: Fri Jul 9 07:07:56 UTC 2021 +### Version: V3.2021.07.2464 +### Updated: Fri Jul 9 08:43:17 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 98951eca4..60e1c578e 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.146 1; +10.1.0.209 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 80715fd4d..a5b5e71d6 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.146 0; +10.1.0.209 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index b145a26e7..4d1c00c0f 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2463 -### Updated: Fri Jul 9 07:07:56 UTC 2021 +### Version: V3.2021.07.2464 +### Updated: Fri Jul 9 08:43:17 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 98951eca4..60e1c578e 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.146 1; +10.1.0.209 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 80715fd4d..a5b5e71d6 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.146 0; +10.1.0.209 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index b145a26e7..4d1c00c0f 100644 --- a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2463 -### Updated: Fri Jul 9 07:07:56 UTC 2021 +### Version: V3.2021.07.2464 +### Updated: Fri Jul 9 08:43:17 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 98951eca4..60e1c578e 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.146 1; +10.1.0.209 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 80715fd4d..a5b5e71d6 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.146 0; +10.1.0.209 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index b145a26e7..4d1c00c0f 100644 --- a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2463 -### Updated: Fri Jul 9 07:07:56 UTC 2021 +### Version: V3.2021.07.2464 +### Updated: Fri Jul 9 08:43:17 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index 6550c57b3..348c5147d 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -2463 +2464 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index b145a26e7..4d1c00c0f 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2463 -### Updated: Fri Jul 9 07:07:56 UTC 2021 +### Version: V3.2021.07.2464 +### Updated: Fri Jul 9 08:43:17 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index 5149f32dd..4e92abccf 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2463 -### Updated: Fri Jul 9 07:07:55 UTC 2021 +### Version: V4.2021.07.2464 +### Updated: Fri Jul 9 08:43:17 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 6e7ee1593..7a1486e6f 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -52.158.247.210 0; +52.250.57.116 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 5149f32dd..4e92abccf 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2463 -### Updated: Fri Jul 9 07:07:55 UTC 2021 +### Version: V4.2021.07.2464 +### Updated: Fri Jul 9 08:43:17 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 98951eca4..60e1c578e 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.146 1; +10.1.0.209 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 80715fd4d..a5b5e71d6 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.146 0; +10.1.0.209 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 5149f32dd..4e92abccf 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2463 -### Updated: Fri Jul 9 07:07:55 UTC 2021 +### Version: V4.2021.07.2464 +### Updated: Fri Jul 9 08:43:17 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 98951eca4..60e1c578e 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.146 1; +10.1.0.209 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 80715fd4d..a5b5e71d6 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.146 0; +10.1.0.209 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 5149f32dd..4e92abccf 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2463 -### Updated: Fri Jul 9 07:07:55 UTC 2021 +### Version: V4.2021.07.2464 +### Updated: Fri Jul 9 08:43:17 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 98951eca4..60e1c578e 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.146 1; +10.1.0.209 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 80715fd4d..a5b5e71d6 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.146 0; +10.1.0.209 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 5149f32dd..4e92abccf 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2463 -### Updated: Fri Jul 9 07:07:55 UTC 2021 +### Version: V4.2021.07.2464 +### Updated: Fri Jul 9 08:43:17 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 6e7ee1593..7a1486e6f 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -52.158.247.210 0; +52.250.57.116 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index b4125f2bc..5a19f38a2 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2463 -### Updated: Fri Jul 9 07:07:55 UTC 2021 +### Version: V4.2021.07.2464 +### Updated: Fri Jul 9 08:43:17 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 6e7ee1593..7a1486e6f 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -52.158.247.210 0; +52.250.57.116 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index c3ce50a41..bc1b89fd4 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2463 -### Updated: Fri Jul 9 07:07:55 UTC 2021 +### Version: V4.2021.07.2464 +### Updated: Fri Jul 9 08:43:17 UTC 2021 ### Bad Referrer Count: 7067 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index 98951eca4..60e1c578e 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.146 1; +10.1.0.209 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 14331f985..bfb43c595 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Abonti -BDCbot -BackDoorBot -FlashGet -MarkMonitor -Offline Explorer +Collector +Evil +IstellaBot +Lanshanbot +MeanPath Bot Re-re -Screaming -Toweyabot -zauba.io +Site Sucker +WEBDAV +WebGo IS +check1.exe diff --git a/dev-tools/test_units/random-bots-for-test.list b/dev-tools/test_units/random-bots-for-test.list index e90febced..fde38db0f 100644 --- a/dev-tools/test_units/random-bots-for-test.list +++ b/dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ +01h4x.com 360Spider 404enemy 80legs -AIBOT -Abonti -AllSubmitter -AlphaBot +ASPSeek +Aboundex +Aboundexbot +Acunetix +AfD-Verbotsverfahren +Aipbot +Anarchie +Anarchy99 Ankit -Anthill -Aspiegel -AwarioRssBot -BDCbot -BackWeb +Asterias +BBBike +BDFetch +BLEXBot +BackStreet Badass Bandit +Barkrowler BatchFTP Bigfoot -BlackWidow -Blackboard -Blow -BlowFish +Bitacle +Black Hole Bolt BotALot -Brandprotect -Brandwatch -Buddy BuiltBotTough -Bullseye BunnySlippers +BuzzSumo +CSHttp Calculon +CazoodleBot +Cegbfeieh +CensysInspect +CherryPicker ChinaClaw Chlooe -Cliqzbot +Claritybot Cogentbot -Copyscape +Collector +CopyRightCheck Craftbot -CrazyWebCrawler Curious CyotekWebCopy DSearch DTS Agent -DatabaseDriverMysqli -Devil -Digincore -Dirbuster -Discobot +Disco +Discoverybot DnyzBot -DomainCrawler -DomainStatsBot +DomCopBot +DomainAppender +DomainSigmaCrawler +Domains Project +Drip +EMail Siphon EMail Wolf Ecxi +EroCrawler +Exabot Express WebPictures -ExtLinksBot Extractor ExtractorPro -Extreme Picture Finder -EyeNetIE +Ezooms +FHscan +FemtosearchBot Firefox/7.0 -FlashGet -Foobot -FrontPage -G-i-g-a-b-o-t +FyberSpider GT::WWW -Genieo -GermCrawler -GetRight +GetWeb Gigablast Go!Zilla -Gotit -Grabber -GrapeshotCrawler +GoZilla +GridBot +HEADMasterSEO HMView -HTTP::Lite -HTTrack -HaosouSpider +Haansoft Harvest -IDBTE4M +Havij +Humanlinks IRLbot -Id-search -IlseBot -Image Sucker -IndeedBot -Indy Library +Image Fetch InfoTekies Intelliseek -Iria +Iskanie +JOC Web Spider JamesBOT Jbrofuzz -Jetty JikeSpider -Joomla -JustView -Jyxobot -LexiBot -Likse -LinkWalker +Keyword Density +LWP::Simple +LeechFTP +LeechGet +Lftp +Libwhisker +LinkScan Linkdexbot -LinkextractorPro -LinkpadBot -LinksManager -LinqiaMetadataDownloaderBot LinqiaScrapeBot -Litemage_walker -Lmspider -MIDown tool -MIIxpc -Mag-Net -Majestic-SEO +Lipperhey +MJ12bot +MQQBrowser +MSIECrawler +Mail.RU_Bot +MarkMonitor Mass Downloader -Masscan -Mata Hari -MauiBot -Mb2345Browser +MeanPath Bot +MegaIndex.ru +Metauri Microsoft Data Access -Mister PiX +Microsoft URL Control Moblie Safari -Morfeus Fucking Scanner -Msrabot -NICErsPRO -NPbot -Nameprotect -Navroad -Nessus -NetAnts -NetLyzer +Mojolicious +Mozlila +Mr.4x3 +Musobot +NearSite +Needle NetSpider NetZIP +Nettrack +Netvibes NextGenSearchBot Nibbler -Nikto -Ninja -Nuclei -OPPO A33 +NimbleCrawler Octopus -Offline Explorer +Offline Navigator OnCrawl -OrangeBot -OrangeSpider -OutclicksBot -OutfoxBot -PECL::HTTP -POE-Component-Client-HTTP -Pandalytics -Panscient +PHPCrawl +Page Analyzer +PageScorer +PageThing.com +Papa Foto Pavuk -PeoplePal -Pi-Monster +Picscout Piepmatz -PleaseCrawl -Pockey -Psbot -PxBroker -Quick-Crawler -RSSingBot +Pump +QueryN Metasearch +RankActive RankActiveLinkBot -RankFlex -Reaper +RankingBot +Rankivabot +ReGet +RebelMouse RedesScrapy RepoMonkey -Scanbot -Scrapy -Screaming -ScreenerBot +SBIder +SISTRIX +SalesIntelligent ScrepyBot -Searchestate SearchmetricsBot +Semrush SeoSiteCheckup Seomoz -SiteExplorer -SiteLockSpider -SiteSnagger -Siteimprove +Siphon +SiteCheckerBotCrawler SmartDownload -Snake -Snapbot -Sogou web spider -Spammen +Snoopy +SocialRankIOBot +Sosospider +Sottopop +SpankBot +Spbot SputnikBot +Sqlmap +Sqlworm Sqworm -Sucker -Sucuri SuperBot Surfbot +SurveyBot Swiftbot -T8Abot -Teleport +Szukacz TeleportPro +Telesoft Telesphoreo -Telesphorep -TheNomad -Toata -Trendiction -Turnitin +TightTwatBot +Toweyabot +Tracemyfile Twice -URLy Warning -URLy.Warning UnisterBot Upflow V-BOT +VB Project Vacuum -VelenPublicWebCrawler -VeriCiteCrawler -Virusdie -WeSEE -Web Collage -Web Fetch +Vagabondo +VidibleScraper +Voltron +WASALive-Bot +WBSearchBot +WEBDAV +WPScan +Wallpapers/3.0 +WallpapersHD +Web Auto +Web Enhancer Web Fuck +Web Pix +Web Sauger WebAuto -WebBandit WebFetch WebGo IS +WebPix +WebSauger WebSucker WebZIP -WebsiteExtractor -Webster +Webalta +WebmasterWorldForumBot +Website Quester +WebsiteQuester +Whacker Whatweb -Wonderbot -Wprecon +Widow +Woobot +Wotbox +Xaldon WebSpider Xenu YoudaoBot +Zade Zauba Zitebot -arquivo-web-crawler -autoemailspider -backlink-check -coccocbot-web +ZmEu +arquivo.pt +check1.exe cognitiveseo -crawler.feedback demandbase-bot domainsproject.org +eCatch +evc-batch +facebookscraper gopher instabid -ips-agent -muhstik-scan +isitwp.com +lwp-trivial +magpie-crawler +meanpathbot +mediawords netEstate NE Crawler -oBot pcBrowser +plumanalytics polaris version +probe-image-size ripz satoristudio.net -scan.lol -seostar -serpstatbot +seoscanners sexsearcher -sitechecker.pro -sogouspider -trendiction.de -webmeup-crawler +spyfu +tAkeOut +voyagerx.com x09Mozilla -zgrab +x22Mozilla +zauba.io diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index af1f1bbdf..a61463e03 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -80legs +360Spider +404enemy ADmantX AIBOT -ASPSeek Aboundex -Aboundexbot -AfD-Verbotsverfahren -AhrefsBot AiHitBot +Aipbot +AllSubmitter Alligator -AlphaBot Anarchy +Ankit Anthill Apexoo -Asterias +Attach AwarioRssBot -BDCbot -BLEXBot +BackDoorBot BackStreet BackWeb -BacklinkCrawler +Backlink-Ceck +Bandit BatchFTP -BetaBot Bigfoot -BlowFish -Bolt +Black Hole +Blow +BotALot +Brandprotect +Brandwatch Buck -BunnySlippers +Buddy BuzzSumo -CCBot -CODE87 -Calculon -CensysInspect -Cliqzbot +CATExplorador Cloud mapping -Cocolyzebot -Cogentbot +Collector CopyRightCheck -Cosmos +Copyscape Craftbot CrazyWebCrawler -Curious CyotekWebCopy -DIIbot DTS Agent -DataCha0s -DatabaseDriverMysqli Demon -Deusu -Devil -Digincore +DigitalPebble Dirbuster -Disco -Discobot DittoSpyder -DomainCrawler -EMail Siphon +DnyzBot +DomCopBot +DomainAppender +DomainSigmaCrawler +DomainStatsBot +Domains Project +Dragonfly +ECCP/1.0 EMail Wolf -EasyDL -Ebingbong -Ecxi -Evil +Exabot Express WebPictures -Extractor +Extreme Picture Finder EyeNetIE -FDM -FemtosearchBot -Fimap -Fuzz +Ezooms +Firefox/7.0 +FlashGet +Freeuploader FyberSpider +Fyrebot +G-i-g-a-b-o-t +GalaxyBot Genieo -GetWeb -Go!Zilla +GetRight +Gigablast +Gigabot Go-Ahead-Got-It -Gotit -GrabNet -GrapeFX -GridBot +Grafula +GrapeshotCrawler +HEADMasterSEO HMView +HTMLparser +HTTP::Lite +Haansoft +HaosouSpider +Harvest +Havij IDBot -IRLbot -IlseBot -InfoTekies -InterGET -Iria +Iblog +IstellaBot +JOC Web Spider +Jbrofuzz Jetty JikeSpider +Joomla +Jorgee Jyxobot -Kenjin Spider +Kinza Kozmosbot -LNSpiderguy -LeechFTP -LexiBot -Lightspeedsystems -Likse -LinkpadBot -Lipperhey Spider +Larbin +LeechGet +Lftp +Libwhisker +LinkScan +Linkdexbot +LinkextractorPro +LinksManager +LinqiaRSSBot +Lmspider Ltx71 -MIDown tool MIIxpc -MQQBrowser +MJ12bot MSFrontPage -MSIECrawler MTRobot Mag-Net -Magnet -Mata Hari +Mass Downloader MauiBot -Mediatoolkitbot -MegaIndex.ru +Meanpathbot Microsoft Data Access -Mister PiX -Mr.4x3 +Microsoft URL Control +Moblie Safari +Mojeek +Morfeus Fucking Scanner +NICErsPRO Name Intelligence +NearSite Net Vampire NetLyzer -NetSpider -Nettrack -Nibbler -Nikto +Netvibes NimbleCrawler -Ninja +Nimbostratus +Nmap +OPPO A33 Octopus -Offline Explorer Offline Navigator -OnCrawl -OpenLinkProfiler +Openfind Openvas -OrangeSpider -PECL::HTTP -Page Analyzer +POE-Component-Client-HTTP +PageAnalyzer +PageGrabber PageScorer -PageThing.com Panscient PeoplePal -Petalbot -Pi-Monster -Picsearch -Piepmatz Pimonster -Pixray -PleaseCrawl -Probethenet -Psbot +Pockey +ProPowerBot +ProWebWalker +Pump +PxBroker PyCurl QueryN Metasearch -Quick-Crawler RSSingBot RankActiveLinkBot RankFlex +RankingBot2 RankurBot +RealDownload Reaper +RebelMouse Recorder -RepoMonkey +RedesScrapy RocketCrawler +Rogerbot +SBIder +SEOlyticsCrawler +SEOprofiler +SISTRIX +SMTBot SalesIntelligent Scanbot +ScoutJet Screaming +ScreenerBot ScrepyBot SearchmetricsBot -SemrushBot -SeoSiteCheckup -SeobilityBot -Shodan -Siphon +Semrush Site Sucker -Siteimprove -Sitevigil -SlySearch -Snake +SiteExplorer +Snapbot +Snoopy Sociscraper -SpankBot +Sosospider SputnikBot -Sqlworm Sqworm +Stripper +Sucuri SuperBot -Suzuran +Surfbot +SurveyBot +T0PHackTeam +T8Abot Teleport -The Intraformant -TheNomad -TightTwatBot +Telesphoreo +Thumbor +Titan +Trendiction Trendictionbot True_Robot -Turingos -TwengaBot -Twice -Typhoeus -UnisterBot -Upflow -V-BOT +Turnitin +TurnitinBot +URLy Warning +URLy.Warning Vacuum -Vagabondo -VelenPublicWebCrawler VeriCiteCrawler -VidibleScraper -Virusdie +Voil +WBSearchBot WEBDAV +WISENutbot WWW-Mechanize -WWW::Mechanize +WWWOFFLE +Wallpapers/3.0 Web Collage -Web Enhancer Web Fetch +Web Fuck +Web Sauger WebAuto -WebCopier WebEnhancer WebFetch -WebFuck -WebImageCollector +WebGo IS +WebLeacher WebSauger WebWhacker -WebZIP +WebmasterWorldForumBot +Webshag WebsiteExtractor -WebsiteQuester Webster Whacker +Whatweb Widow -WinHTTrack -WiseGuys Robot -Wprecon +Wonderbot +Xenu +Zermelo +Zeus Zitebot ZmEu -ZoomBot ZoominfoBot -check1.exe +ZumBot +archive.org_bot +arquivo.pt +backlink-check cognitiveseo crawl.sogou.com crawler4j +demandbase-bot eCatch -evc-batch instabid -internetVista monitor isitwp.com -lwp-request -meanpathbot -oBot -page scorer -pcBrowser +magpie-crawler +muhstik-scan plumanalytics -polaris version ripz -seobility -seostar -serpstatbot +s1z.ru +siteripz sogouspider -sp_auditbot -spyfu -sysscan -tAkeOut -trendiction.com +webmeup-crawler x22Mozilla +xpymep1.exe +zauba.io diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 8b696e27f..b4733ed7e 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -adtech.fr -conboy.us -dekoration.us -freegamesplay.online -landscapebackgrounds.blogspot.com -offtime.ru -onlyforemont.ru -rightenergysolutions.com.au -svnuppsalaorebro.se -videtubs.pl +bupropion-sr-150-mg.us +cialgenisrx.com +gamerscorps.com +gateway.zscalertwo.net +indo-export.ru +marcowebonyodziezowe.pl +markjaybeefractal.com +seofied.com +staticfs.host +worldlovers.ru diff --git a/dev-tools/test_units/random-referrers-for-test.list b/dev-tools/test_units/random-referrers-for-test.list index e50c4bef7..5e526b0cd 100644 --- a/dev-tools/test_units/random-referrers-for-test.list +++ b/dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ 03p.info +0c47f8422d3f.com 1-99seo.com -1pamm.ru -24x7-server-support.site -4webmasters.org -5-steps-to-start-business.com -51unlim.ru -7a2rnuey1tw9ar.ru -a96527gi.beget.tech -abctoppictures.net -acortarurl.es -actulite.com -adsfresh.men -adultactioncam.com -advertex.info -afmuseum.com -agreda.pluto.ro -ahmedabadwebs.com -alienwheels.de -all-streaming-media.com -alt-servis.ru -androids-store.com -animal-drawings.com -animalia-life.club -anjalika.co.in -anti-virus-removal.info -aosexkontakte.net -apartment.ru -apifasterlightin-a.akamaihd.net -apimountainbikei-a.akamaihd.net -apioasisspacenet-a.akamaihd.net -apisecretsaucebi-a.akamaihd.net -apple.com-webbrowsing-security.review -apple.com-webbrowsing-security.science +12bet.com +256bit.by +2ads.co.uk +2itech.ru +300richmond.co.nz +34.gs +3dracergames.com +3xforum.ro +404.mn +7xc4n.com +a1security.com.ua +aa9bd78f328a6a41279d0fad0a88df1901.com +aadroid.net +academia-nsk.org +addbin.men +addblueoff.com.ua +akvamaster.dp.ua +alborzan.com +alevinefesleri.com +aliexpresscashback.club +alif-ba-ta.com +all4invest.info +am-se.com +amt-k.ru +angkortours.vn +anlimebel.kiev.ua +anticrawler.org +apartmentratings.com +apiappenableinfo-a.akamaihd.net +apimegabrowsebiz-a.akamaihd.net +app5.letmacworkfaster.world +approved.su +arabgirls.adultgalls.com arabseyes.com -as5000.com -asdfg.pro -asmxsatadriverin.aircus.com -asrv-a.akamoihd.net -asscenihotosidea.blogspot.co.za -atout-energie-69.com -atovh.local-finders.com -auto-moto-elektronika.cz -autolombard-krasnodar.ru -automobile-spec.com -autovideobroadcast.com -autowebmarket.com.ua -aviav.eu -avtovolop.ru -awaybird.ru -ayakino.net -baltgem.com -beauty-b0x.pl -berlininsl.com -bestwebsitesawards.com -beta.hotkeys.com -bjgugu.net.ua -bloke.com -bmsco.co -bonanza-fortune.men -boo-arts.com -brendbutik.ru -brianhenry.co -buchananshardware.com -businesxxl.com -buyessayonline19.blogspot.ru -cactussoft.biz -cam-kontakte.org -candycrushshop.com -canoncdriverq3.pen.io -cartierring.xyz -casablancamanor.co.za -casinopinup-wins.com -cbox.ws -centrdebut.ru -certifywebsite.win -cheapmoncler.xyz -chelyabinsk.xrus.org -chinacheapelitejerseys.com -chinajerseyswholesalecoupons.com -chiptuninger.com -chocolateslim-u-srbiji.com -christianlouboutinreplica.pw -ciekawinki.pl -cjmc.info -clayvasedesigns.tk -coin-hive.com -com-supportcenter.website -cometorussia.net -comissionka.net -companies-catalog.com -compliance-brian.xyz -compliance-checker-7.info -compliance-george.top -consorzioilmosaico.org -cordstrap.cc -craftburg.ru -creditnation.ru -cyclobenzaprinevstizanidine.blogspot.com -dafatiri.com -ddsoldes.fr +arcteryxstore.online +arenanews.com.ua +artpicso.com +asophoto.com +auto4style.ru +autogrep.ru +autotrends.today +aviav.org +avkzarabotok.com +azbukafree.com +bablonow.ru +back.dog +balitouroffice.com +banan.tv +bangla-chat-uk-paralud.ga +bankmib.ru +bannerconnect.net +batanga.net +best-deals-products.com +bestcalovebracelet.cn +bestcoin.cc +bestcurs.org +bestmouthwash.club +bestpornuha.com +bg6s0.com +biteg.xyz +bizlist.com.de +bizru.info +blogrankers.com +boazpower.com +bonky.biz +bonvillan.com +boobsimge.com +bookmark4you.com +bristolhostel.com +buigas.00it.com +burger-tycoon.com +businesn.men +buy-meds24.com +buyhoverboard.com +buyk.host.sk +bycontext.com +c1.onedmp.com +canacopegdl.com +cardul.ru +carson.getenjoyment.net +cbcseward.com +cdnanalytics.xyz +celebsopics.com +centrumcoachingu.com +chatroulette.life +chatroulette.online +cheapbelstaff.online +check-this-out-now.online +checkm8.com +chocolateslim-en-france.com +christianlouboutinsaleonline.us +citetick.com +cleanallvirus.com +coaching-netz.info +coindirect.io +compliance-margo.top +coolingoods.com +coolyarddecorations.com +coslab.club +cracksplanet.com +cranly.net +cymbaltaandtizanidine.blogspot.com +cyse.tk +d2jsp.org +data1.scopich.com +davebestdeals.com dealitright.click -decoratinghomes.ga +delfin-aqua.com.ua +deluxedumps.com +demian.kz demon-tweeks.com -detalizaciya-tut.biz -devochki.top -directivepub.com +depositfiles-porn.ga +digital-video-processing.com +digitalassetmanagement.site +dimkino.ru +distver.ru djonwatch.ru -dlya-android.org -dnsrsearch.com -domain2008.com -domaincheck.io -doreenblog.online -dprtb.com +doctormakarova.ru +dojki365.online +dood.live +dostavimvdom.ru +dostavka-v-krym.com dptaughtme.com -drunkenstepfather.com -easycommerce.cf -easync.io -easyshoppermac.com -easyukraine.com -econom.co -electricwheelchairsarea.com -elektrischezigarettekaufen2.cowblog.fr -elektronischezigarettekaufen1.beeplog.com -elektrozigarette1.dreamwidth.org +druzhininevgeniy63.blogspot.com +dvd-famille.com +dyshagi.ru +eaplay.ru +ebooknovel.club +ec-file.info +ege-essay.ru +el-nation.com +elektir.ru +elektrischezi.canalblog.com +elektrischezigaretten2.beeplog.com +elexies.info elmacho.xyz eng-lyrics.com -enthuse.computernetworksonline.com -eonpal.com -ero-advertising.com -erotikstories.ru -erotische-geschichten-xxl.com -etotupo.ru -eupornstar.info -evehemming.blogspot.com.au -event-tracking.com -exoclick.com -extlabs.io -extremal-blog.com -extremepornos.net -eyelike.com.ua -ezigaretteshop.webs.com -f07.de -f1nder.org -facebook-mobile.xyz -failingmarriege.blogspot.com -fanoboi.com -fashionavenuegame.com -fastfixing.tech -filmi-onlain.info -find1friend.com -fodelsedagspresenter.nu -frankofficial.ru -freenode.info -freeuploader.ml -frivgame250.com +ewebarticle.info +exchangeit.gq +expdom.com +expresstoplivo.ru +ezigarettenkaufen1.deviantart.com +falcoware.com +falllow.gq +favornews.com +feargames.ru +floridamhca.org +floridamobilebillboards.com +footballfarrago.com +forex-indextop20.ru +forum20.smailik.org +foto-basa.com +frameimage.org +fullfileaccess.com +funnel.co.za fusoradio.info -gambarkatabaru.com -gamedayassist.com -gamerextra.com -games.kolossale.ru -gaytube.com -generic-pills-online.com -geoads.com -getmiro.com -gheus.altervista.org -ghostvisitor.com -globalsurfari.com -gonextmedia.com -goodhumor24.com -gopixdatabase.com -gosmeb.ru -graphicwe.org -greatgrace.ru -gtopstats.com -hawaiisurf.com -healthcarestore.info -highstairs-a.akamaihd.net -hmywwogw.bloger.index.hr -home-task.com -homedo.fabpage.com -hoporno.com -hotkeys.com -hotsocialz.com -hoverboard360.nl +fym.com.ru +gambarkataku.co +gaz-voshod.ru +gdebestkupit.ru +genericviagrasildenafiled.net +gentamicineyedrops.blogspot.com +gepezz.info +getoutofdebtfree.org +gigablast.com +glastecfilms.com.my +global-ics.co.za +gratuitbaise.com +grizzlysgrill.com +groupmoney.ru +guiadeserraazul.com +hackers-crackers.tk +hacktougroup.ru +hamptonoaks.ca +hauleddes.com +hawaiielectriclight.com +hchha.com +hdpixent.com +hermesbracelets.xyz +hetmanship.xyz +heygidday.biz +hidefiles.org +hit-kino.ru +hobbyhuren24.net +homelygarden.com +hop.clickbank.net +hotenergy.ru +hothot.ru +housediz.com +houtings.xyz howmuchdoestizanidinecost.blogspot.com -hptwaakw.blog.fc2.com -hully.altervista.org -iboss.com -id-forex.com -ido3.com -ilikevitaly.com -imgarcade.com -imk.com.ua -impresagaia.it -infokonkurs.ru -insta-add.pro -intellekt21.ru -ipornox.xxx -irkutsk.zrus.org -istizanidinelikexanax.blogspot.com -iwantmyfreecash.com -jackpotchances.com -janavibekken.no -japan-bearings.ru -javcoast.com -jewelryandfiligree.com -jikoman.info -jjbabskoe.ru -jrpmakati.com -juliadiets.com -kakablog.net -karusel-market.ru +hybrid.ru +icaseclub.ru +idealtits.net +igtools.club +ilmexico.com +imicrovision.com +immobiliaremassaro.com +incep.imagine-ex.co +intellektmedia.at +intermesh.net +investzalog.ru +inzn.ru +ispac.org +itis4you.com +ixora.pro +jennyfire.ru +jo24news.com +johnrobertsoninc.com +jwcialislrt.com +kalandranis.gr kazan.xrus.org -kazan.zrus.org -keki.info -keywordteam.net -kino-ecran.ru -kino-filmi.com -kiprinform.com -kit-opt.ru -kliksaya.com -koronirealestate.gr -krasivoe-hd.com -krasivye-devushki.net -krasula.pp.ua -ktotut.net +keywordsuggest.org +kinohit1.ru +komputers-best.ru +konkursowo-24.pl +konteiner24.com +koversite.info kupit-adenu.ru -kvartir-remont.biz -labelwater.se -lacapilla.info -lakomka.com.ua -law-check-nine.xyz -layola.biz.tc -lidoradio.com -liquimondo.com -littleberry.ru +kurwa.win +lanasshop.ru +larose.jb2c.me +law-check-two.xyz +lawyersinfo.org +ldrtrack.com +lechenie-gemorroya.com +ledpolice.ru +lenpipet.ru +letmacworkfaster.site +lifehacĸer.com +limtu.ifmo.ru +linkredirect.biz littlesunraiser.com -livingroomdecor.info -ljusihus.se -louboutinreplica.pw -louboutinreplica.xyz -luckyshop.net.ua -machicon-ueno.info -mackeeper-land-672695126.us-east-1.elb.amazonaws.com -mahnwachen-helfen.info -makenahartlin.com -maladot.com -marmitaco.cat -marmotstore.online -mb140.ru -mecash.ru -medispainstitute.com.au -metaxalonevstizanidine.blogspot.com -mhtr.be -millionare.com -minharevisao.com -mirmedinfo.ru -mjchamonix.org -mlf.hordo.win -mobioffertrck.com -monclerjacketsoutlet.win +live-sexcam.tk +localmatchbook.com +lodki-pvh.dp.ua +magazin-pics.ru +magnetpress.sk +makis.nu +managerpak204.weebly.com +maranbrinfo.com.br +marblestyle.ru +maridan.com.ua +masturbate.co.uk +mature.free-websites.com +mbiologi.ru +mebelcomplekt.ru +mediafresh.online +mediaoffers.click +medicinacom.ru +medkletki.ru +metallosajding.ru +mexicotravelnet.com +michaelkorsoutlet.store +mikozstop.com +mindeyegames.com +mir-betting.ru +misandesign.se +misswell.net +mistr-x.org +mmofreegames.online +mmostrike.ru +modenamebel.ru +mokrayakiska.com +mole.pluto.ro monetizer.com-01.site -monitorwebsitespeed.com -monsterdivx.tv -monthlywinners.com -moroccosurfadventures.com -movpod.in -mowser.com -mrvideospornogratis.xxx -mttwtrack.com -musicspire.online -mwtpludn.review -mycouponizemac.com -myfreetutorials.com -mykings.pw -nalogovyy-kodeks.ru -namenectar.com -nash-krym.info +moneyviking-a.akamaihd.net +mosrif.ru +most.gov.iq +mouselink.co +mrbitsandbytes.com +mrpornogratis.xxx +musezone.ru +mydirtystuff.com +mydoctorok.ru +myphotopipe.com +naperehresti.info +narkologiya-peterburg.ru +naturalbreakthroughsresearch.com +naturalshair.site naturtreenspicerx.pw -niki-mlt.ru -nl.netlog.com -nomuos.it -nootrino.com -nowtorrents.com -nudo.ca -nufaq.com -nullrefer.com -o-o-6-o-o.com -obiavo.net -of-ireland.info -oivcvx.website -oktube.ru -omgtnc.com -omsk.xrus.org +nbsproject.ru +neobux-bg.info +newstudio.tv +nightvision746.weebly.com +nikitsyringedrivelg.pen.io +normalegal.ru +nuit-artisanale.com +oakridgemo.com +oconto.ru +ok.ru +onefilms.net +onemantrip.com +onlinadverts.com online-podarki.com -online7777.com -onlinemegax.com -optitrade24.com +onlineporno.site +onlineserialy.ru +onlinetvseries.me +oralsexfilme.net +ordernorxx.com orsonet.ru -ostroike.org -outrageousdeal-a.akamaihd.net -ozas.net -panchro.xyz -papasdelivery.ru -pdns.cz -pensplan4u.com -pewit.pw -phrcialiled.com -phuketscreen.com -picphotos.net -pictures-and-images.net -pigrafix.at -pinapchik.com +osb.se11.ru +osoznanie-narkotikam.net +ownshop.cf +p-business.ru +parlament.biz +partner-stop.men +partybunny.ru +pendelprognos.se +penisvergrotendepillennl.ovh +pestomou.info +pharmacyincity.com +phelissota.xyz +pics-group.com +picture-group.com +pirateday.ru +pix24x7.com pizza-tycoon.com -pladform.ru -pogodnyyeavarii.gq -poisk-zakona.ru -popmarker.com -pornokajf.com -portadd.men -postclass.com -potolokelekor.ru -pozdravleniya-c.ru -pozdrawleniya.ru -privat-girl.net -prizesbook.online -professionalwritingservices15.blogspot.ru -projefrio.com.br -pron.pro -prostitutki-belgoroda.org -prostitutki-novgoroda.org -prostitutki-tolyatti.org -purplesphere.in -qualitymarketzone.com -quizzitch.net -qwertty.net -radiodigital.co -rednise.com -refererx.com -remont-komputerov-notebook.ru -reyel1985.webnode.fr -rialp.getenjoyment.net -ring4rhino.com -ringporno.com -rostov.xrus.org -rubbed.us -s-forum.biz -saitevpatorii.com -samara.rosfirm.ru -santasgift.ml -savingsslider-a.akamaihd.net -sbricur.com +pl-vouchers.com +plastgran.com +plastgranar.nu +poem-paying.gq +poems.com.ua +popugaychiki.com +porno-raskazy.ru +pornoplen.com +pornosemki.info +pr-ten.de +prohoster.info +prombudpostach.com.ua +pronekut.com +pushdata.sendpulse.com +puzo2arbuza.ru +pyramidlitho.webs.com +qwesa.ru +r-control.ru +rabotaetvse.ru +ranksonic.com +ranksonic.info +realmonte.net +reelheroes.net +registry-clean-up.net +rennlist.com +rique.host.sk +rocis.site +roleforum.ru +roll123.com +royalads.net +rusoft-zone.ru +sa-rewards.co.za +sajatvelemeny.com +saneitconsulting.com +scalerite.co.za scanner-george.top -scanner-irvin.top -scrapy.org -scripted.com -seansonline24.pl -searchengineranker.email +scanner-ivan.top +scat.porn +scrnet.biz.ua +se.bnt-team.com +search.1and1.com +searchkut.com +searchlock.com searchtooknow-a.akamaihd.net -searchwebknow-a.akamaihd.net -secretscook.ru -sei80.com -serpstat.com +seasaltwithfood.com +securityallianceservices.com +see-your-website-here.com +sel-hoz.com +sendearnings.com +seo-tools-optimizing.com +serptehnika.ru service-core.ru sevendays.com.ua -sex-dating.co -sexkontakteao.info -sexreliz.com -sexs-foto.com -sextracker.be -sfj-ror.no -shakhtar-doneck.ru -shtaketniki.ru -similardeals.net -site-speed-check.site -siteheart.net -sizeplus.work -skladvaz.ru -sky-mine.ru -slomm.ru -slutloadlive.com -sneakyboy.com -social-vestnik.ru -socialsignals24.com -softtor.com -songplanet.ru -sotkal.lark.ru -space-worry.ml -space4updating.win -spbchampionat.ru -sport-video-obzor.ru -srdrvp.com -sstroy44.ru -starpages.net -stathat.com -stats-public.grammarly.io -stpolice.com -strv.se -superfish.com -surfbuyermac.com -sweepstakes.rewardit.com -szucs.ru -tattooreligion.ru -taxi-v-eisk.ru -tedxrj.com -teletype.in -theallgirlarcade.com -thebluffs.com -thecoral.com.br -thegalerie.eu -timeallnews.ru -timer4web.com -tizanidinedosage.blogspot.com -tizanidineforopiatewithdrawal.blogspot.com -tizanidinehcl2mg.blogspot.com -tmtrck.com -tomatis.gospartner.com -toon-families.com -toondinsey.com -tooplay.com -top10-online-games.com -torrentdownloadhub.com -torture.ml -toxicwap.com -toys.erolove.in -tracxn.com -traffic2money.com -trafficmonetize.org -tsyndicate.com -ugogo.info +sevenstars7.com +sex-sex-sex5.com +sexvideo-sex.com +silverage.ru +silvermature.net +simple-share-buttons.com +skylta.com +soft-program.com +soundfrost.org +spl63.fr +spywarebegone.com +starwars.wikia.com +stoki.ru +streetfire.net +stroyhelp-dv.ru +stroymonolit.su +stuff-about-money.com +superkanpo.com +svtrd.com +swinger-mobil.net +sygraem.com +tanieaukcje.com.pl +tasteidea.com +tattoo33.ru +tdsing.ru +techart24.com +testingads.pro +thedownloadfromwarez.blogspot.com +thegamerznetwork.com +thegolfclub.info +thewomenlife.com +thin.me.pn +tiens2010.ru +titan-ads.life +titelhelden.eu +tizanidine4mgstreetvalue.blogspot.com +tizanidine4mguses.blogspot.com +tizanidinecapsules.blogspot.com +tizanidineduringpregnancy.blogspot.com +tizanidineingredients.blogspot.com +tkanorganizma.ru +top-study.work +topquality.cf +topsy.com +torrent-newgames.com +torrentred.games +totu.us +toy-shop.top +tozup.com +traffictrade.life +trainoffend.ml +tri-slona.org +trudogolik.net +trustedhealthtips.com +tsstcorpcddvdwshbbdriverfb.aircus.com +tube8.com +twelvevisionspartyofcolorado.com +twojebook.pl +twu.com.ua +uggbootsoutletsale.us uk-zheu20.ru -ukkelberg.no -ukr-lit.com -ultimatesetnewfreeallsoftupgradesystems.pw -unionmarkt.de -uogonline.com -uptime-as.net -urengoy.pro -urll.eu -utrolive.ru -velen.io -velobikestock.com -viagengrarx.com -viagra-soft.ru -viddyoze.com -video-chat.cn -videochat.life -videositename.com -vip-parfumeria.ru +ukrtvir.com.ua +um-razum.ru +unlimitdocs.net +urccvfmc.bloger.index.hr +v24s.net +valuado.com +vekzdorov.ru +veselokloun.ru +videokrik.net +vigrx-original.ru +vilingstore.net +villacoloniale.com +vintontech.info +vip2ch.com +vipsexfinders.com +virus-respirators.com +visionwell.com.cn +vkmusics.ru vksaver-all.ru -vksex.ru -vltai.com -volume-pills.biz -voluumtracker1.com -wallpaperaccess.com -waterefficiency.co -webstatsdomain.org -welcomeauto.ru -westen-v.life -westen-z.life -whengirlsgowild.com -wikes.20fr.com -wnhjavlhezp.gq -woman-orgasm.ru -word-vorlagen.net -works.if.ua +voltrknc1.com +vseigru.one +vseuznaem.com +watchinf.com +wcb.su +webjam.com +website-speed-checker.site +website-stealer.nufaq.com +westermarkanjou.se +whatistizanidinehclusedfor.blogspot.com +wheelchairliftsarea.com +wholesalejerseychinashop.com +wholesalenfljerseys.us.com +witclub.info +withstandingheartwarming.com +wjgony.com +womensterritory.ru +word-vorlagen.xyz +wordpresscore.com +wstroika.ru wtsindia.in -wufak.com -www.event-tracking.com -www.pinnacle-bets.com -wzgyyq.com -x-musics.com -x-porno.video -xcombear.ru -xev.ru -xgftnlrt.bloger.index.hr -xlovecam.com -xmlinde.com +www.souvenirua.com +xblog.in +xingzi-vision.com +xn----8sblgmbj1a1bk8l.xn----161-4vemb6cjl7anbaea3afninj.xn--p1ai +xn----9sbebi2bvzr7h.xn--p1ai +xn----ctbbcjd3dbsehgi.xn--p1ai +xn----ctbigni3aj4h.xn--p1ai +xn--1-8sbcpb0bdm8k6a.xn--p1ai +xn--80aanaardaperhcem4a6i.com xn--80aebbcbcdemfkhba4byaehoejh8dza3v.xn--p1ai -xn--80atua3d.xn--p1ai -xnxx699.com -xxxguitars.com -yaaknaa.info -yaminecraft.ru -yellowfootprints.com -yougame.biz -zaimite.ru -zapatosenventa.info -zastenchivosti.net -zastroyka.org -zeroredirect12.com -zeroredirect7.com -zionstar.net -zixizop.net.ru -znakomstva-piter78.ru -zolotoy-lis.ru -zoolubimets.ru +xolodremont.ru +yebocasino.co.za +youporn-ru.com +your-bearings.com +youradulthosting.com +yourothersite.com +zakazvzloma.com +zawyna.ua +zebradudka.com +zeroredirect11.com +zhacker.net +znaniyapolza.ru +zone-kev717.info zrizvtrnpale.tk -zverokruh-shop.cz +zynax.ua diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index bf900d2f4..638e9ca6b 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -1webmaster.ml -24-ak.ru -2nt.ru -4istoshop.com -7zap.com -a.pr-cy.ru -adcash.com -adclickthru.net -addray.pro -adpremium.org -adrenalin-stk.ru -affiliate-fr.com +000free.us +007angels.com +1000n1.ru +1j7740kd.website +2ads.co.uk +2drittel.de +34.gs +3xforum.ro +40cg.com +add-add.men +adultactioncam.com +agapovdg.ru agecheckadult.com -alert-fjg.xyz -all-streaming-media.com -allproblog.com -aludecor.info -am-se.com -anatomiy.com +ahrntal.verymes.xyz +alborzan.com +alert-fdm.xyz +aliexpresscashback.club android4fun.org -apartment.ru -apiallgeniusinfo-a.akamaihd.net -apifasterlightin-a.akamaihd.net -asrv-a.akamaihd.net -autodo.info -autoloans.com -aviav.ru.com -avkzarabotok.com -azinoofficial777.ru -azlex.uz -babs.com.ua -balla-boo.se -begalka.xyz -beget.tech -besttorrentknifta.weebly.com -betteroffers.review -bird1.ru -bloggers.nl -boobsimge.com -bookhome.info -c1.onedmp.com -celejihad.info -cenoval.ru -centrdebut.ru -chlooe.com -chocolateslim-u-srbiji.com -choosecuisine.com -cialischstgerts.com -cinemaenergy-hd.ru -click2pawn.com -club-musics.ru -computernetworksonline.com -comsysnet.com -corta.co -cscwtalkto.site -custom-product-labels.com -cutt.us -dailyfinancefix.com -db.speedup.gdn -ddsoldes.fr -detmebel.su -diamond-necklace.info -dmmspy.com -dojki365.online +apartmentratings.com +asrvrep-a.akamaihd.net +ayakino.net +b00kmarks.com +bambi.ck.ua +beclean-nn.ru +benchmarkcommunications.co.uk +bigames.online +bimmerpost.com +bingo8888.com +blog.yam.com +bloglag.com +boleznikogi.com +bravegirlsclub.com +bristolhotel.com.ua +browsepulse-a.akamaihd.net +calstaterealty.us +candy-glam-hp.com +cashonads.com +catalogs-parts.com +cheapwebsitehoster.com +cialischmrx.com +cleanersoft.com +clubfashionista.com +cognitiveseo.com +colobit.biz +compliance-elena.xyz +compliance-mary.top +cookie-law-enforcement-ff.xyz +copro.pw +d2jsp.org +darodar.com +delayreferat.ru +dneprsvet.com.ua +doctormakarova.ru domain-submit.info +domain.webkeyit.com +domaincrawler.com domaineaneblanc.com -doyouknowtheword-flummox.ml -druzhbany.ru -e705.net -elitesportsadvisor.com -englishdictionaryfree.com -enskedesquashclub.se -epngo.bz -exdocsfiles.com -ezigarettenkaufen2.dreamwidth.org -farmprofi.net -fbdownloader.com -fickkontaktehobbyhuren.com -floating-share-buttons.com -foto-basa.com -froggytube.com -fungirlsgames.net -gallerylisting.com -games.kolossale.ru -gdzkurokam.ru -generalporn.org -getpopunder.com -gq-catalog.gq -green-tea.tv -gsbs.com.ua -hacktougroup.ru -hamacapty.com -hccoder.info -heartofbeijing.blogspot.com -hesteel.pl -hjaoopoa.top -home.myplaycity.com -hoverboardforsaledirect.com -hptwaakw.blog.fc2.com -ifirestarter.ru -ilovevitaly.com -imitex-plus.ru -inet-traffic.com -infodocsportal.com -internetartfair.com -jerseywholesalebizchina.com -jikoman.info -junketjuice.blogspot.com -jyrxd.com -kchaxton.com -kimcurlrvsms.com -kinoduh.ru -kolotiloff.ru -koptims.tiu.ru -kw21.org -lescinq.com -lidoradio.com +dpihatinh.gov.vn +draniki.org +earl-brown.info +eas-seo.com +eldiariodeguadalajara.com +elektrischezi.canalblog.com +empirepoker.com +enews.tech +enge-fotzen.info +erotische-geschichten-xxl.com +europeanwatches.ru +evehemming.blogspot.com.au +evening-dating-club.info +extremepornos.net +eyelike.com.ua +facebook-mobile.xyz +factorynightclub.com +familienzahnaerzte.com +fastcrawl.com +filefilter.weebly.com +financepoints.eu +fiverr.com +flashbannernow.com +flowersbazar.com +franklinfire.co +free-share-buttons.top +freemaintenancesysforpcandmac.top +front.to +ftns.ru +gamedayhouse.com +get-your-social-buttons.info +ggiaro.com +gktt.ru +glall.ru +gobongo.info +graphid.com +habermetre.com +hawaiisurf.com +hem.passagen.se +hostingclub.lk +i-service.kz +ibmdatamanagement.co +ido3.com +ilovevitaly.xyz +investpamm.ru +isitpaleo.info +isotoner.com +istanbulit.com +j-times.ru +jerseychinabizwholesale.com +johnnyhaley.top +justucalling32211123456789.tk +jwcialislrt.com +kanimage.com +kevblog.top +kidd.reunionwatch.com +kostenloser-sex.com +kountrylife.com +krasnodar.xrus.org +krasula.pp.ua +kunstaktien.info +lacasamorett.com +ladov.ru +lechtaczka.net +letslowbefast.site +limtu.ifmo.ru lineavabit.it -lockerz.com -mahnwachen-helfen.info +lyrics.home-task.com mapquestz.us -masterseek.com -matsdale.com -mcadamssupplyco.com -militarysale.pro -mnogolok.info -monetizer.com-01.site -morocco-nomad-excursions.com +medicines-choice.com +meget.co.za +mirzonru.net +mobile-appster.ru +mobilierland.com +mobsfun.net +moje-recenze.cz +mompussy.net +moncleroutletonline.xyz +money-every-day.com +moviemail-online.co.uk moyakuhnia.ru -music7s.me -my-bc.ru -myshopmatemac.com -national-today-winning-winner.club -naturtreenspicerx.pw -news-readers.ru -no-fuel.org -notasprensa.info -novosti-hi-tech.ru -nowtorrents.com -nudejapan.net -obiavo.kz -obnal.org -okayimage.com -oralsexfilme.net -ourville.info +mrmoneymustache.com +my-cash-bot.co +nalogovyykodeks.ru +neks.info +nfljerseyscheapchinabiz.com +nhl09.ru +nicefloor.co.uk +notfastfood.ru +noumeda.com +odywpjtw.bloger.index.hr +oecnhs.info +offer.wpsecurity.website +omgtnc.com +onemactrckr.com +onlinemeetingnow.com +orion-code-access.net +orsonet.ru +ossmalta.com +ownshop.cf oxotl.com -panicatack.com -parajumpersstore.online -partner-print.men -partner-stop.men -pearlisland.ru -percin.biz.ly -poem-paying.gq -positive2b.ru -priceg.com -prize44.com -purplesphere.in -pxhdwsm.com -queerspace.com -razvratnoe.org -real-time-analytics.com -redbottomheels.xyz -rednise.com -reklamuss.ru -relax.ru -remont-mobile-phones.ru -renecaovillasale.online +parvezmia.xyz +pinsdaddy.com +pl.id-forex.com +pornhub-ru.com +porno-chaman.info +pornofiljmi.com +poshiv-chehol.ru +posible.net +powc.r.ca.d.sendibm2.com +prchecker.info +preparevideosafesystem4unow.space +pro-okis.ru +profitsport.club +proxyelite.biz +psoriasis-file.trade +registry-clean-up.net +remybutler.fr rentalcarnavi.info -riralmolamsaca.tk -royal-betting.net -rutor.group -salmonfishingsacramentoriver.com -samoiedo.it -sanjosestartups.com -sbricur.com -scenarii-1-sentyabrya.uroki.org.ua -see-your-website-here.com -service.adtech.fr -sexkontakteao.info -shareyards.com -shariki-zuma-lines.ru -shiksabd.com -shopvilleroyboch.com.ua -sigmund-freud.co.uk -silvermature.net -simpoed.ufop.br +rentaremotecomputer.com +rep-am.com +rff-cfal.info +russintv.fr +sajatvelemeny.com +sakhboard.ru +santasgift.ml +sardinie.us +savingsslider-a.akamaihd.net +sbwealthsolutions.ca +score-ads.men +search.1and1.com +search.pch.com +secretscook.ru +semaltmedia.com +seminarygeorgia59.ga +seorankinglinks.com +servingnotice.com +sexkontakte-seite.com +sexobzor.info +sexpornotales.com +shop-electron.ru sisiynas.ru -site-speed-checker.site -slowmacfaster.trade -smartpet.ru -smartshoppymac.com -soft1.ru -speedup-my.site -spin2016.cf -sr-rekneskap.no -sribno.net -strangeduckfilms.com -stroilka.info -sugarkun.com -takeflyte.com -tako3.com -tdsing.ru +sitiz.club +slujbauborki.ru +snjatie-geroinovoy-lomki.ru +social-buttons.com +space-worry.ml +spasswelt.net +stop-gepatit.te.ua +stretchingabuckblog.com +sugvant.ru +sundrugstore.com +supermama.top +suralink.com +td-l-market.ru +tech4master.com testbotprocessor44.com -thebitcoincode.com -thebluenoodle.com -thefarmergame.com -thegameriders.com -thewebsitetemplate.info -thfox.com -titan-ads.life -tizanidineandcipro.blogspot.com -tizanidineandhydrocodone.blogspot.com -tizanidinehcl2mgtablet.blogspot.com -tizanidinewithdrawal.blogspot.com -toon-families.com -top-deal.com.pl -torture.ml -toys.erolove.in -trkdf.com -tryrating.com -twiclub.in -twittruth.com -ukkala.xyz -unblocksit.es -v720hd.ru -vacances-voyages.info -vchulkah.net -ventelnos.com -via-energy-cumpara.com -videosbox.ru -vsesubwaysurfers.com -vseuznaem.com -walkme.com -we-are-gamers.com -webadvance.club -webinstantservice.com -wholesalecheapjerseysfree.com -www.solartek.ru -xitjw.info -xn--80adaggc5bdhlfamsfdij4p7b.xn--p1ai -xn--80aebbcbcdemfkhba4byaehoejh8dza3v.xn--p1ai -xvideosbay.com -xxxhdvideo.site -yaaknaa.info -yaminecraft.ru -yourads.website -znakom.sibtest.ru -zoodrawings.com -zrelaya.pw -zvooq.eu +thegalerie.eu +tizanidinehcl4mgtab.blogspot.com +tizanidinesleepaid.blogspot.com +torrentgamer.net +unrealcommander.org +urlcut.ru +us-america.ru +usadacha.net +ustion.ru +varbergsvind.se +vedomstvo.net +veerotech.com +viagraneggrx.com +videonsk.com +visitcambridge.org +vod.com.ua +vrotike.ru +webshoppermac.com +website-speed-checker.site +webstatsdomain.org +wetgames.ru +wholesalejerseychinaoutlet.com +wholesalejerseyscheapjerseys.us.com +wineration.com +winwotgold.pl +wpthemedetector.co.uk +ww2awards.info +www.888.com +x-diesel.org +xn--80ahdheogk5l.xn--p1ai +yoursite.com +youtoner.it +zeroredirect12.com +znaniyapolza.ru +zrizvtrnpale.tk +zzbroya.com.ua diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index 80715fd4d..a5b5e71d6 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.146 0; +10.1.0.209 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index 0af768dc0..9f5790908 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -16222,3 +16222,1251 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2464 +### Updated: Fri Jul 9 08:43:18 UTC 2021 +### Bad Bot Count: 615 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: OPPO A33 +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ + diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 0af768dc0..9f5790908 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -16222,3 +16222,1251 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2464 +### Updated: Fri Jul 9 08:43:18 UTC 2021 +### Bad Bot Count: 615 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: OPPO A33 +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ +