From b372155e7f6f5eff4420e5816f1a6618521b63b4 Mon Sep 17 00:00:00 2001 From: mitchellkrogza Date: Fri, 30 Jul 2021 14:14:04 +0000 Subject: [PATCH] V4.2021.07.2492 [ci skip] --- AUTO-CONFIGURATION.md | 2 +- MANUAL-CONFIGURATION.md | 2 +- README.md | 2 +- _google_analytics_ghost_spam/README.md | 2 +- .../etc/nginx/conf.d/globalblacklist.conf | 4 +- conf.d/globalblacklist-testing-version.conf | 4 +- conf.d/globalblacklist.conf | 4 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- dev-tools/buildnumber | 2 +- dev-tools/globalblacklist-testing.template | 4 +- dev-tools/globalblacklist.template | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- dev-tools/test_units/blacklist-ips.conf | 2 +- .../random-bots-for-test-quick.list | 20 +- .../test_units/random-bots-for-test.list | 304 ++-- .../random-bots-for-whitelist-test.list | 302 ++-- .../random-referrers-for-test-quick.list | 20 +- .../test_units/random-referrers-for-test.list | 934 ++++++------ .../random-referrers-for-whitelist-test.list | 478 +++---- dev-tools/test_units/whitelist-ips.conf | 2 +- dev-tools/tmp/robots.txt | 1252 +++++++++++++++++ robots.txt/robots.txt | 1252 +++++++++++++++++ 45 files changed, 3585 insertions(+), 1081 deletions(-) diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index f704f8207..4dc104b59 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2491 +#### Version: V4.2021.07.2492 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [617](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 474fc797f..0e0f91f2f 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2491 +#### Version: V4.2021.07.2492 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [617](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/README.md b/README.md index e7491cdab..e63457aab 100755 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2021.07.2491 +#### Version: V4.2021.07.2492 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [617](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 9ecc6c4ea..df5185804 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,7 +3,7 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2021.07.2491 +#### Version: V4.2021.07.2492 #### Bad Referrers Blocked: [7068](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [617](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 4f7783f32..e8058ec4d 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2491 -### Updated: Thu Jul 29 14:07:29 UTC 2021 +### Version: V4.2021.07.2492 +### Updated: Fri Jul 30 14:08:14 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 617 ################################################### diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index c05eb3caf..a7ffbeaa6 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2491 -### Updated: Thu Jul 29 14:07:29 UTC 2021 +### Version: V3.2021.07.2492 +### Updated: Fri Jul 30 14:08:14 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 617 ################################################### diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 4f7783f32..e8058ec4d 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2491 -### Updated: Thu Jul 29 14:07:29 UTC 2021 +### Version: V4.2021.07.2492 +### Updated: Fri Jul 30 14:08:14 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf index c05eb3caf..a7ffbeaa6 100644 --- a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2491 -### Updated: Thu Jul 29 14:07:29 UTC 2021 +### Version: V3.2021.07.2492 +### Updated: Fri Jul 30 14:08:14 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 656f6577d..bb96131fd 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.55 1; +10.1.1.54 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index bc1a013ac..54446da99 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.55 0; +10.1.1.54 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index c05eb3caf..a7ffbeaa6 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2491 -### Updated: Thu Jul 29 14:07:29 UTC 2021 +### Version: V3.2021.07.2492 +### Updated: Fri Jul 30 14:08:14 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 656f6577d..bb96131fd 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.55 1; +10.1.1.54 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index bc1a013ac..54446da99 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.55 0; +10.1.1.54 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index c05eb3caf..a7ffbeaa6 100644 --- a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2491 -### Updated: Thu Jul 29 14:07:29 UTC 2021 +### Version: V3.2021.07.2492 +### Updated: Fri Jul 30 14:08:14 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 656f6577d..bb96131fd 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.55 1; +10.1.1.54 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index bc1a013ac..54446da99 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.55 0; +10.1.1.54 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index c05eb3caf..a7ffbeaa6 100644 --- a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2491 -### Updated: Thu Jul 29 14:07:29 UTC 2021 +### Version: V3.2021.07.2492 +### Updated: Fri Jul 30 14:08:14 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index 951ba797e..59780dc7b 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -2491 +2492 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index c05eb3caf..a7ffbeaa6 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2491 -### Updated: Thu Jul 29 14:07:29 UTC 2021 +### Version: V3.2021.07.2492 +### Updated: Fri Jul 30 14:08:14 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index 4f7783f32..e8058ec4d 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2491 -### Updated: Thu Jul 29 14:07:29 UTC 2021 +### Version: V4.2021.07.2492 +### Updated: Fri Jul 30 14:08:14 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 7fc603fe0..8f9360366 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -40.125.66.133 0; +137.117.65.139 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 4f7783f32..e8058ec4d 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2491 -### Updated: Thu Jul 29 14:07:29 UTC 2021 +### Version: V4.2021.07.2492 +### Updated: Fri Jul 30 14:08:14 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 656f6577d..bb96131fd 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.55 1; +10.1.1.54 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index bc1a013ac..54446da99 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.55 0; +10.1.1.54 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 4f7783f32..e8058ec4d 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2491 -### Updated: Thu Jul 29 14:07:29 UTC 2021 +### Version: V4.2021.07.2492 +### Updated: Fri Jul 30 14:08:14 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 656f6577d..bb96131fd 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.55 1; +10.1.1.54 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index bc1a013ac..54446da99 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.55 0; +10.1.1.54 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 4f7783f32..e8058ec4d 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2491 -### Updated: Thu Jul 29 14:07:29 UTC 2021 +### Version: V4.2021.07.2492 +### Updated: Fri Jul 30 14:08:14 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 656f6577d..bb96131fd 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.55 1; +10.1.1.54 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index bc1a013ac..54446da99 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.55 0; +10.1.1.54 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 4f7783f32..e8058ec4d 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2491 -### Updated: Thu Jul 29 14:07:29 UTC 2021 +### Version: V4.2021.07.2492 +### Updated: Fri Jul 30 14:08:14 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 7fc603fe0..8f9360366 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -40.125.66.133 0; +137.117.65.139 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 635ee10da..8d13706b6 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2491 -### Updated: Thu Jul 29 14:07:29 UTC 2021 +### Version: V4.2021.07.2492 +### Updated: Fri Jul 30 14:08:14 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 7fc603fe0..8f9360366 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -40.125.66.133 0; +137.117.65.139 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index cb4e0009c..1a04d6f26 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2491 -### Updated: Thu Jul 29 14:07:29 UTC 2021 +### Version: V4.2021.07.2492 +### Updated: Fri Jul 30 14:08:14 UTC 2021 ### Bad Referrer Count: 7068 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index 656f6577d..bb96131fd 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.55 1; +10.1.1.54 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 02c2d454a..b548f2fca 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -AiHitBot -Anarchy99 -Jyxobot -Linkdexbot -RankingBot -Sqworm -Steeler -WebSauger -Who.is Bot -com.plumanalytics +Battleztar Bazinga +Dirbuster +Extractor +Lipperhey Spider +NetZIP +Pimonster +Pixray +SISTRIX +SuperHTTP +Surfbot diff --git a/dev-tools/test_units/random-bots-for-test.list b/dev-tools/test_units/random-bots-for-test.list index ef3763833..11ad34d3b 100644 --- a/dev-tools/test_units/random-bots-for-test.list +++ b/dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ +01h4x.com 360Spider -404checker 404enemy 80legs -ADmantX -ASPSeek +AIBOT Abonti -Aboundexbot +Aboundex +Acunetix +AfD-Verbotsverfahren +AhrefsBot AiHitBot -Anarchy99 -Anthill +Alexibot +Anarchie +Anarchy +Ankit Apexoo -Aspiegel Asterias -AwarioRssBot -AwarioSmartBot -BBBike -BDCbot -BackDoorBot -BacklinkCrawler +BDFetch +BackStreet Badass Bandit -Barkrowler -Bigfoot -Bitacle +BatchFTP +BetaBot Blow BlowFish +Boardreader Bolt -Brandwatch -Bullseye +BotALot +Brandprotect +BuiltWith +BunnySlippers +CATExplorador +CSHttp Calculon CazoodleBot Cegbfeieh CheeseBot -Cogentbot -CopyRightCheck -DIIbot -DSearch -DTS Agent +ChinaClaw +Chlooe +Cliqzbot +Collector +Copyscape +Curious +Custo Demon -Deusu -Devil Digincore -DigitalPebble Disco Discobot -DittoSpyder -DomCopBot -DomainAppender -Domains Project +DomainCrawler +DomainSigmaCrawler +DomainStatsBot Dotbot -ECCP/1.0 -EMail Siphon +Drip EMail Wolf -Ebingbong -Ecxi -Evil -Exabot -ExtractorPro -EyeNetIE -Ezooms -FDM +EasyDL FHscan Fimap -Flunky -Freeuploader -FrontPage -GT::WWW +FlashGet +FyberSpider +Fyrebot +G-i-g-a-b-o-t +GalaxyBot +Genieo GetRight GetWeb -Getintent +Gigablast Go-Ahead-Got-It GoZilla GrabNet -Grabber -GrapeFX GridBot -HTMLparser +HEADMasterSEO +HTTP::Lite +Haansoft +HaosouSpider Heritrix Hloader -Humanlinks -HybridBot +IDBTE4M +IRLbot Iblog -Image Fetch -Indy Library Intelliseek -Internet Ninja +Iria IstellaBot -JOC Web Spider +Jetty JikeSpider -Jorgee -JustView Jyxobot -Keyword Density -Kinza LNSpiderguy +Lanshanbot Larbin -LeechFTP +Leap +Lightspeedsystems Linkdexbot -LinkextractorPro -LinkpadBot -Lipperhey -Lmspider -MFC_Tear_Sample -MSFrontPage -MSIECrawler -Magnet -Mail.RU_Bot -Majestic SEO -Majestic-SEO -Majestic12 -MegaIndex.ru -MicroMessenger +LinksManager +LinqiaMetadataDownloaderBot +LinqiaScrapeBot +Litemage_walker +MIIxpc +MJ12bot +MQQBrowser +Mass Downloader +Mata Hari +MeanPath Bot +Mediatoolkitbot +Microsoft URL Control Mojeek -Mr.4x3 -Msrabot +Mojolicious +Mozlila Musobot -NICErsPRO -Nessus -Net Vampire -NetAnts -NetLyzer +Name Intelligence +Nameprotect +NearSite +Needle NetSpider -NetZIP Netcraft -Netvibes +Nibbler Niki-bot -Nikto -Nimbostratus +NimbleCrawler Ninja +Octopus +Offline Navigator +OnCrawl OpenLinkProfiler -OrangeBot OrangeSpider -OutclicksBot -OutfoxBot PECL::HTTP -POE-Component-Client-HTTP -PageGrabber +PHPCrawl PageScorer -PageThing.com Pandalytics -Pavuk PeoplePal -Piepmatz -Pockey -ProPowerBot +Petalbot +Picscout +PictureFinder +Pixray +PleaseCrawl +ProWebWalker +Psbot Pu_iN +Pump PxBroker -Quick-Crawler +PyCurl RSSingBot RankActive -RankActiveLinkBot RankFlex -RankingBot2 -RankurBot -Re-re -RealDownload -RocketCrawler -SEOkicks -SEOlyticsCrawler -SEOprofiler +Rankivabot +Recorder +Ripper +Rogerbot +SEOkicks-Robot SEOstats +SISTRIX +SMTBot ScanAlert Scanbot -Screaming +ScoutJet +Scrapy ScreenerBot Searchestate SearchmetricsBot Semrush -SemrushBot +SentiBot SeobilityBot -Siphon -SiteCheckerBotCrawler +Seomoz +Site Sucker SiteExplorer -SiteSnagger -Sitebeam -Siteimprove -SocialRankIOBot -Sosospider -Spammen -Sqlmap +SiteLockSpider +SiteSucker +Sitevigil +SlySearch +SmartDownload +Snapbot +Sociscraper +SpaceBison +Spanner +Sqlworm Steeler Sucker Sucuri -SuperBot SuperHTTP +Surfbot +SurveyBot Suzuran T0PHackTeam -TheNomad +T8Abot +Teleport +Telesoft +Telesphorep +TightTwatBot +Titan Tracemyfile Trendiction Trendictionbot -Turingos -Turnitin +TurnitinBot TwengaBot -URLy Warning -Upflow -VB Project +Twice +URLy.Warning +Vagabondo VidibleScraper -WASALive-Bot -WBSearchBot -WISENutbot -WPScan -WWW-Collector-E -WWW-Mechanize +Virusdie +VoidEYE +WEBDAV WallpapersHD -Web Enhancer -Web Fuck -Web Pix +WeSEE +Web Auto +Web Fetch Web Sucker WebBandit -WebCollage -WebEnhancer -WebFetch -WebReaper -WebSauger +WebCopier +WebFuck +WebGo IS +WebPix WebStripper WebSucker -Webshag +WebmasterWorldForumBot +Website Quester WebsiteExtractor +WebsiteQuester Whack -Whacker -Whatweb +Who.is Bot +Widow WinHTTrack -WiseGuys Robot Wonderbot -Wprecon -Xaldon_WebSpider +Xenu +YoudaoBot +Zermelo Zeus +Zitebot ZumBot +arquivo.pt +autoemailspider +check1.exe coccocbot-web com.plumanalytics crawl.sogou.com -crawler4j -eCatch -evc-batch heritrix -internetVista monitor ips-agent -lwp-request magpie-crawler meanpathbot -netEstate NE Crawler +mediawords +page scorer plumanalytics -ripz -s1z.ru +probe-image-size +scan.lol +seoscanners +seostar +serpstatbot sexsearcher -trendiction.com +sitechecker.pro +sp_auditbot +trendiction.de +ubermetrics-technologies.com webmeup-crawler -x09Mozilla +zgrab diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 281a3f5ca..231401b97 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ +ADmantX AIBOT -Aboundex -AiHitBot -Anarchy99 -Anthill -Apexoo -Attach -AwarioRssBot +Abonti +AllSubmitter +Alligator +AlphaBot +Anarchie +Aspiegel +Asterias AwarioSmartBot -BBBike BackStreet -Badass +Backlink-Ceck Bandit Battleztar Bazinga +Blackboard Blow -BlowFish -CODE87 -Calculon +Bolt +BotALot +Brandwatch +BuiltBotTough +BuzzSumo +CazoodleBot +Cegbfeieh CensysInspect -ChinaClaw -Chlooe -Claritybot +CherryPicker +Cliqzbot Cloud mapping -Cocolyzebot -CopyRightCheck -Copyscape -Cosmos -CrazyWebCrawler -Curious +Cogentbot +Collector +Copier +Crescent +DIIbot DatabaseDriverMysqli -Demon -Digincore +Devil DigitalPebble -Discobot Dispatch +DnyzBot DomCopBot -DomainAppender -Domains Project -Dotbot -Download Wonder -Dragonfly -Drip -Ecxi -EirGrabber -EroCrawler +DomainSigmaCrawler +DomainStatsBot +EMail Siphon +EMail Wolf +EasyDL +Evil Exabot -Express WebPictures -Extractor +ExtractorPro +Extreme Picture Finder +EyeNetIE Ezooms -FDM FHscan +FemtosearchBot Firefox/7.0 -FlashGet Flunky -Foobot Freeuploader -FrontPage -Fuzz -FyberSpider -Genieo GermCrawler -Gigablast +Getintent +Gigabot +Go-Ahead-Got-It +GoZilla Gotit +Grabber GrapeshotCrawler -HTMLparser HTTP::Lite -HaosouSpider +Harvest Havij -Hloader -HybridBot IDBTE4M Iblog IlseBot +Image Fetch Image Sucker -IndeedBot +Indy Library InfoNaviRobot +InfoTekies InterGET -Iria -JamesBOT -JennyBot -Joomla +Internet Ninja +IstellaBot +JOC Web Spider +Jbrofuzz +Jetty +JikeSpider +Jorgee JustView +Kenjin Spider Keyword Density -Kinza +LNSpiderguy Lanshanbot Leap -Libwhisker -Lightspeedsystems +LeechGet +LieBaoFast +LinkWalker Linkbot -LinkextractorPro LinkpadBot LinksManager Lipperhey -Litemage_walker -Lmspider -MFC_Tear_Sample +Ltx71 +MIDown tool +MJ12bot +MQQBrowser +MSFrontPage +MSIECrawler MTRobot Mag-Net -Magnet -Majestic SEO -Majestic-SEO +Majestic12 +MarkMonitor +Mass Downloader +Mata Hari Mb2345Browser -Mediatoolkitbot -Microsoft Data Access -Microsoft URL Control -Moblie Safari +Meanpathbot +Mister PiX +Mojeek Morfeus Fucking Scanner Mozlila -Mr.4x3 Musobot -NPbot -NearSite +NICErsPRO +Name Intelligence +Nameprotect +Navroad Nessus +NetAnts +NetLyzer NetSpider NetZIP -Nibbler +NextGenSearchBot Nimbostratus +Ninja Nmap -Offline Explorer -OrangeSpider -OutclicksBot +Nutch +Octopus +Offline Navigator +OpenLinkProfiler +Openvas +OrangeBot OutfoxBot -PECL::HTTP -PHPCrawl -Page Analyzer -PageGrabber -PageScorer +PageAnalyzer +PageThing.com +Pandalytics +Papa Foto +Pavuk PictureFinder +Piepmatz +Pimonster Pixray -PleaseCrawl -Probethenet +Pockey +ProPowerBot Pu_iN -PxBroker -RSSingBot +RankActive RankFlex Rankivabot -RankurBot -Re-re +RealDownload Reaper RebelMouse -Recorder +RedesScrapy +RepoMonkey Ripper RocketCrawler -SBIder -SEOkicks SEOlyticsCrawler -SEOprofiler +SEOstats +SISTRIX SMTBot -ScanAlert -Scrapy -Screaming -ScreenerBot +ScoutJet Searchestate -SearchmetricsBot Semrush -SentiBot -SeobilityBot -SiteCheckerBotCrawler +SeoSiteCheckup +Seomoz +Site Sucker SiteExplorer +SiteSnagger +Sitebeam Sitevigil SlySearch SmartDownload -Snapbot -Sosospider -Spanner -Spinn3r +Sociscraper +SpankBot +Spbot SputnikBot -Sqworm -Swiftbot +Sqlworm +Steeler +Sucker +SuperHTTP +Surfbot +SurveyBot Szukacz -T0PHackTeam -T8Abot -Teleport +TeleportPro Telesoft -Telesphorep -TheNomad -TightTwatBot -Titan Toata -Tracemyfile -Trendictionbot -True_Robot -TurnitinBot +Turnitin Twice -URLy Warning -Upflow -Vagabondo -VeriCiteCrawler -Virusdie +UnisterBot +Vacuum +VelenPublicWebCrawler VoidEYE -Voil -Voltron WASALive-Bot WBSearchBot WEBDAV -WPScan WWW-Collector-E -WWW-Mechanize WWW::Mechanize -Wallpapers/3.0 -Web Auto -Web Enhancer -Web Sauger -WebBandit +Web Collage +Web Fetch +Web Fuck +Web Sucker WebCollage -WebEnhancer +WebCopier WebFetch WebFuck +WebGo IS WebImageCollector WebLeacher WebPix -WebReaper +WebStripper WebSucker WebWhacker -WebZIP -WebmasterWorldForumBot -Webshag +Webalta +WebsiteExtractor WebsiteQuester -Widow -Woobot +Whacker +Who.is Bot +WiseGuys Robot Wotbox +Wprecon +Xenu YoudaoBot Zauba -Zeus +Zitebot +ZmEu ZoomBot -ZumBot ZyBorg -arquivo-web-crawler -autoemailspider -crawl.sogou.com -crawler.feedback +arquivo.pt +backlink-check +check1.exe +cognitiveseo +com.plumanalytics domainsproject.org evc-batch facebookscraper -gopher heritrix -lwp-request -magpie-crawler -netEstate NE Crawler -plumanalytics -polaris version -scan.lol -seoscanners +isitwp.com +lwp-trivial +meanpathbot +satoristudio.net +scalaj-http +seobility +seostar +serpstatbot +sexsearcher sitechecker.pro siteripz -sp_auditbot spyfu -webmeup-crawler -x09Mozilla +voyagerx.com +xpymep1.exe zauba.io diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 7128ae723..757cfb43c 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -4pp13.com -babs.com.ua -buzz-porno.info -com-onlinesupport.site -dipstar.org -dojki-hd.com -louboutinreplica.pw -meet-flirt-dating.com -pro-okis.ru -top10-online-games.com +affrh2015.com +alfapro.ru +community.allhiphop.com +galaxy-family.ru +jus0wil.pp.ua +microsearch.ru +paydayloanslocal.com +picture-group.com +porno.simple-image.com.ua +tasteidea.com diff --git a/dev-tools/test_units/random-referrers-for-test.list b/dev-tools/test_units/random-referrers-for-test.list index 02b5613fa..be13b191b 100644 --- a/dev-tools/test_units/random-referrers-for-test.list +++ b/dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -000free.us -0912701309f8ce.com -0daymusic.org -101lesbian.xyz -10bet.com -12bet.com -178evakuator178.ru -1adult.com -1flag.co.za -2daytrendingnews.com -4pp13.com -51.la +1-99seo.com +18ps.ru +1qingdao.com +21h2o.com +34.gs +4ureyesonly.com 70casino.online -7fon.ru -7ooo.ru -883zy.com -91abcw.com -acads.net +7makemoneyonline.com +7search.com +8lufu.com +abcdeg.xyz acc.eu.org -actionnooz.com -actulite.com -adminshop.com +addray.pro ads-seo.men -adsref.men -adultgalls.com -adviceforum.info -adzpower.com -agapovdg.ru -airmaxshoes-2016.com -akuhni.by -alfapro.ru -allcredits.su -allnews.md -allsilver925.co.il -amateurgalls.com -amyfoxfitness.com -angel1777.kz -anime.dougasouko.com -anticrawler.org -aosheng-tech.com -apibetweenlinesn-a.akamaihd.net +adsfresh.men +adtech.de +adultnet.in +adverster.com +advertisingtag.net +adzerg.com +aibolita.com +alfa-img.com +alldubai.biz +ameblo.jp +android-systems.ru apimegabrowsebiz-a.akamaihd.net -apispringsmartne-a.akamaihd.net -apparel-offer.com -appsaurus.com -aproposde.com -aquarium-pro.ru -arcadepages.com -artavenuegardenstudios.com -asacopaco.tk -atmagroup.ru -australia-opening-times.com -autobudpostach.club -autotop.com.ua -avramstroy.ru +apiqualitinknet-a.akamaihd.net +apisecretsaucebi-a.akamaihd.net +app-ready.xyz +applyneedy.xyz +arcteryxsale.online +arpelsreplica.xyz +asmxsatadriverin.aircus.com +atleticpharm.org +auto.rusvile.lt +avtochehli.by azartmix.com +azartmsl.com +azartniy-bonus.com +azbukadiets.ru +babespcs.com +babs.com.ua +back.dog backlink4u.net -bamo.xsl.pt -bannerspace.com -bannerswap.com -bavsac.com -bdsmgalls.net -beauty-bracelet.com -berdasovivan.ru -best-coupon-offer.com -bestofpicture.com -bestprofits-there.com +baladur.ru +banki76.ru +basisches-wasser.net +beachpics.com +bedandbreakfast.com +bestbuy.ca +bestdooz.com +bestdraws.com +bestserials.com +bestwebsiteawards.com bet-prognoz.com -biglistofwebsites.com -bimatoprost-careprost.com -bio-market.kz -bio.trade-jp.net +big-cash.net bitcoin-ua.top -bonzibuddi.com -brakehawk.com -buckspar.gq -business-suggested.tk -buyantiviralwp.com -c-gainsbourg.com -cacheimages.com -callcafe.info -calvet.altervista.org -candiceloves.us -canrioloadm.gq -capture-room.com -cardiosport.com.ua -carmuffler.net -castingbank.ru -caulderwoodkennels.com -cejewelry.xyz -cercacamion.it -chatroulette.video -chiblackhawks-jerseys.com -cialiswithoutadoctor.net -clksite.com -club-samodelkin.ru -compliance-fred.top -compliance-george.xyz -compliance-jane.top -compliance-mary.top -concealthyself.com +bitcoinremote.com +biteg.xyz +blobar.org +blog-hits.com +bloggedporn.com +bloggen.be +bluerobot.info +boanonihaca.tk +buyk.host.sk +buynorxx.com +canacopegdl.com +cand.jp +candytiner.com +cardsdumps.com +carloans.com +carson.getenjoyment.net +cascadelink.org +cdnnetwok.xyz +cenoval.ru +chidporn.com +cibpenonptib.flu.cc +cjs.com.ru +classiquebijoux.ru +clodo.ru +cmd.kz +compliance-nelson.top +compliance-viktor.top conocer-sanabria.com -contentsexpress.com -cookie-law-enforcement-ee.xyz -coolgamechannel.com -coprofam.org -coslab.club -crest3d.ru -dayibiao.com -delta-line.men -demian.kz -deniven.1bb.ru -dentfidemountpreach.tk -diarioaconcagua.com +cookingmeat.ru +coolwallpapers4k.info +copyrightclaims.org +counterbot.com +covetnica.com +cranly.net +cupday.com +d2jsp.org +darkbooks.org +data-mining.tk +data1.scopich.com +dataloading.net +decoratinghomes.ga +defenderxtactical.com +dgfitness.co dimaka.info -divku.ru -djekxa.ru -domination.ml -dosugrostov.site -download-walpaperhd.blogspot.com -downvids.net -dvd-famille.com +dimfour.com +dipstar.org +dividendo.ru +doktoronline.no +doublepimp.com +downloaddy.net +driving.kiev.ua +drugspowerstore.com +dtnlyss.com +dyt.net +e-commerce-seo.com e2click.com -easyukraine.com -egvar.net -electronicadirect.com -elektrozigarette2.wordpressy.pl -englishdictionaryfree.com -erotik-kostenlos.net -erotiktreff24.info -esfchat.tk -eshop4u.jp -este-line.com.ua -exonline.info -extremepornos.net -ezigarettenshop1.yolasite.com -f5mtrack.com -falllow.gq -family1st.ca -faracontrol.ir -fashion-stickers.ru -favornews.com +earthmagic.info +ed-shop01.ru +educatemetv.com +efkt.jp +ekn-art.se +ekosmetyki.net.pl +eleimgo.pw +elektrischezigarette1.onsugar.com +elektrischezigarettekaufen2.cowblog.fr +elektroniksigaraankara.info +elektronischezigarettekaufen2.tumblr.com +emerson-rus.ru +epicbrogaming.com +erotikchat-24.com +es5.com +escortplius.com +exchangeit.gq +exoclick.com +extremal-blog.com +extremez.net +extstat.com +ezigarettekaufen2.mpbloggar.se +familyholiday.ml fefo.gdn -ferrotodo.com -fertilitetsradgivningen.se -fickblock18.com -fickenprivat.info -filerockstar298.weebly.com +fickkontakte.org +fickkontaktehobbyhuren.com +fidelityfunding.com +fighrofacciufreesig.ga +filesdatabase.net +fillmewithhappiness.com +film-one.ru +filmi-onlain.info financeloan.us -financepoints.eu -findpik.com -flex4launch.ru -forensicpsychiatry.ru -forodvd.com -foxweber.com -fuckbuddybestgilf.info -fuel-gas.com -gamebackyard.com -gamewrath.com -gateway.zscalertwo.net -gcup.ru -gelezki.com -gemara.com -gembird.com -gfaq.ru -girlgamerdaily.com -girls-ufa.ru -gk-atlant.info -glavtral.ru -go2mike.ru -golmau.host.sk -googlefeud.com -goosefishpost.bid -gox.com.ua -gpms.org.my -gq-catalog.gq -graphid.com -gsasearchengineranker.site -guiadeserraazul.com -guod.me -handicapvantoday.com -healing-dysplasia.ru -healthcarestore.info -heatpower.ru -heliko.no -hentaiheroes.com -herokuapp.com -heroz.fr -hexpilot.com -himgaws.pw -homemature.net -hotel-mkad.ru -hoterika.com -hothot.ru -houseofgaga.ru -houston-vikings.com -i-hobot.ru -i-service.kz -igrovyeavtomaty777.ru -ikritikimou.gr -ilikevitaly.com -ilmexico.com -imguramx.pw -intellego.info -investsuccess.org -irkutsk.zrus.org -islamtoday.co.za -ispaniya-costa-blanca.ru -istizanidineanopiate.blogspot.com -itag.pw -ivanstroi.ru -ixora.pro -javidol.site -javitas.info -javxxx18.com -jetsli.de -jovencitas.gratis +findacheaplawyers.com +findanysex.com +fingerprintjs.com +firesub.pl +firstdrugmall.ru +floridamobilebillboards.com +forum.doctissimo.fr +foxinsocks.ru +freakycheats.com +free-floating-buttons.com +freelotto.com +freevpn.space +frivgame250.com +ftns.ru +fullfileaccess.com +fun-mobi.pl +funponsel.com +fz139.ttk.ru +galaxyflowers.ru +gameplexcity.com +gayxperience.com +geileweiber.tk +genetworx.com +get-seo-domain.com +gidonline.one +greatidea.marketing +grtyi.com +gstatey.net +guigyverpo.cf +gwebtools.com.br +gyffu.com +hazardky.net +hccoder.info +hdimagegallery.net +hermesbracelets.xyz +hikesearch.net +hotenergy.ru +housemilan.ru +howlongdoestizanidinestayinyoursystem.blogspot.com +hscsscotland.com +humanorightswatch.org +i-midias.net.br +ibmdatamanagement.co +ideibiznesa2015.ru +igithab.com +imagine-ex.co +imitex-plus.ru +indetiske.ya.ru +instasexyblog.com +intermesh.net +ipostroika.ru +iqs.biz.ua +iscblog.info +istanbulit.com +ivearchenceinflu.cf +iweblist.info +janerikholst.se +javcoast.com +johnnyhaley.top kandidos.com -kareliatobacco.ru -karpun-iris.ru +karadene.com +kazrent.com keywordhouse.com -kichenaid.ru -kicknights.gq -kihi.gdn -kimcurlrvsms.com -kinoflux.net -kinosed.net -kinozapas.org -kiskinhouse.com -kladrus.ru -konkursov.net -koopilka.com -kotaku.com -ks1234.com -kurbappeal.info -labafydjxa.su -lafriore.ru -landreferat.ru -lapitec.eu -laptoper.net -lechenie-gemorroya.com -levaquin750mg.blogspot.com +keywordsdoctor.com +kinopolet.net +kinostorm.net +klejonka.info +klumba55.ru +kongoultry.net +kozhniebolezni.com +kraljeva-sutjeska.com +krasnodar.zrus.org +ku6.com +kw21.org +lawyers.cafe +leftofcentrist.com +libertybilisim.com +lifehacĸer.com +light.ifmo.ru lineavabit.it -linkpulse.com -littleberry.ru -llastbuy.ru -loginduepunti.it -losangeles-ads.com +linkdebrideur.xyz +lirunet.ru +littlesexdolls.com +longadventure.com +lowephotos.info ltvperf.com -lulea-auktionsverk.se -luxup.ru -lyrics.home-task.com -m1media.net -m3gadownload.pl -macfix.life -maglid.ru -manipulyator-peterburg.ru -mapquestz.us -marcoislandvacations.net -markjaybeefractal.com -martlinker.com +m4ever.net +macdamaged.tech +mactechinfo.info +magnetic-bracelets.ru +maju.bluesalt.co +mall.uk +malls.com +mansparskats.com +manve.info +marcogrup.com maslenka.kz -matematikus.info -maxxtor.eu -mcar.in.ua -mediawhirl.net -medkritika.ru -medmajor.ru -meds-online24.com -members.ghanaweb.com -memberty.com -mensandals.xyz -metascephe.com -metaxalonevstizanidine.blogspot.com -mexicosleevegastrectomy.com -mhtr.be +master-muznachas.ru +masturbate.co.uk +mathgym.com.au +mebel-ekb.com +mebeldekor.com.ua +medicineseasybuy.com +mega-bony2017.pl +metarip.ru michellblog.online -microsoftportal.net -microstatic.pl +microsearch.ru minecraft-rus.org -mini-modus.ru -mirmedinfo.ru -mixx.com -mnogolok.info -mojowhois.com -mompussy.net -moncleroutletonline.xyz -mp3ritm.top -mrvideospornogratis.xxx -mrwhite.biz -muizre.ru -musicvidz.ru -muznachas-service.ru -mycaf.it -mylovelibrabry.com -nakrutka.cc -narosty.com -natali-forex.com -negociosdasha.com -nelc.edu.eg -net-radar.com -netpics.org -nfljerseys.online -nicovideo.jp -niroo.info -nl.netlog.com -nofreezingmac.work +minneapoliscopiers.com +missclub.info +missvietnam.org +mixtapetorrent.com +mobifunapp.weebly.com +mobstarr.com +moesonce.com +monclerjacketsoutlet.win +monetizer.com-01.site +moneytop.ru +moneyzzz.ru +moscow.xrus.org +mossmesi.com +mototsikl.org +moxo.com +mrcsa.com.au +mrmoneymustache.com +msfsaar.de +mtmtv.info +mydeathspace.com +myfreemp3.eu +mygameplus.ru +myiptest.com +mykings.pw +mymercy.info +myphotopipe.com +nalogovyykodeks.ru +netfacet.net +netoil.no +newfilmsonline.ru +next-dentists.tk +nextrent-crimea.ru +nibbler.silktide.com +nonews.co +nudepatch.net +o-o-6-o-o.ru o.light.d0t.ru -old-rock.com -openfrost.net -ops.picscout.com +oddamzadarmo.eu +of-ireland.info +okel.co +okuos.com +onclickpredictiv.com +onefilms.net +onlainbesplatno.ru +onlinadverts.com +online-zaymy.ru +onlinebay.ru +ooo-olni.ru +ooomeru.ru +opinionreelle.com +orel-reshka.net orenburg-gsm.ru -outrageousdeal-a.akamaihd.net -oxotl.com -pacificair.com -paradontozanet.ru -pattersonsweb.com -pawli.eu -pc-virus-d0l92j2.pw -pendelprognos.se -perso.wanadoo.es -pewit.pw -photokitchendesign.com +orion-code-access.net +osoznanie-narkotikam.net +ourville.info +oxford-book.com.ua +paceform.com +painting-planet.com +pammik.ru +paparazzistudios.com.au +parajumpersjakkesalgnorge.info +partner-trustworthy.men +pearlisland.ru +pflexads.com +photo.houseofgaga.ru photosaga.info -php-market.ru -phrcialiled.com -pictures-and-images.net -pigrafix.at +physfunc.ru +picturesify.com +picurams.pw +pierrehardysale.online +pinapchik.com +pinkduck.ga +pipki.r.acdnpro.com pisanieprac.info -pixshark.com -pl-vouchers.com -plastweb.ru -polska-poezja.com -pons-presse.com -poptool.net -porngalleries.top -pornophoto.xyz -pornstartits.xblog.in +plastjulgranar.se +pojdelo.weebly.com +pokemongooo.ml +pokerniydom.ru +polimga.pw +popcash.net +porno-dojki.net +pornoblood.com +pornoinn.com +pornoklad.ru +pornotubexxx.name +pornozhara.com +pornpost.in +portside.cc +postclass.com potoideas.us -pozdravleniya-c.ru prchecker.info -princevc.com -pro-poly.ru -proekt-gaz.ru +preconnubial.usuby.site +pregnant.guru +privacyassistant.net +pro-tec.kz profitsport.club -prostitutki-kharkova.org -pufip.com -puzo2arbuza.ru -qualitymarketzone.com -quebec-bin.com -ranksays.com -rating-casino2021.ru -recipedays.com -redirect.trafficreceiver.club -reimageplus.com -replica-watch.ru -resellerclub.com -ritlweb.com -rootandroid.org -rosbalt.com.ua -rybalka-opt.ru -s-forum.biz -sarahmilne.top -sbprabooks.com -sbricur.com -scanner-brian.top -scanner-jane.top -scanner-willy.top -scansafe.net -scenicmissouri.us -scmor.ilxc.cc -scrapy.org -screentoolkit.com -search.pch.com -searchimage.co -searchimpression.com -seasaltwithfood.com -seinterface.com -seo-tools-optimizing.com -seo18.su -seoanalyses.com -seoholding.com -serdcenebolit.com -service.adtech.us -sevenstars7.com -sex.hotblog.top -sexxdate.net +purplestats.com +pyramidlitho.webs.com +pyrodesigns.com.au +qcstrtvt.bloger.index.hr +rainbowice.ru +randalljhoward.com +rapevideosmovies.com +rapidokbrain.com +rcb101.ru +realmonte.net +realtytimes.com +registrysweeper.com +renhacklids.tk +replicaclub.ru +ribieiendom.no +ric.info +ring4rhino.com +runtnc.net +rusenvironmental.net +s1z.ru +sad-torg.com.ua +sampleletters.net +sarm.tk +sbwealthsolutions.ca +se-welding.ru +searchwebknow-a.akamaihd.net +seasonvar.ru +selectads.men +sell-fb-group-here.com +seo-prof1.xyz +seoboxes.com +seogadget.ru +seorankinglinks.com +sex-tracker.de +sex-znakomstva.online +sexad.net +sexcamamateurchat.com +sexphoto.site sexy-pings.com -sexytrend.ru -sfd-chess.ru -sharebutton.net -sharebutton.to -shemale-sex.net -sherlock.se -shmetall.com.ua -shopcheermakeup.info -sim-service.net -simon3.ru -siteaero.com -smadihome.com -small-games.biz -snip.to -snjack.info -snomer1.ru -social-fun.ru -solicita.info -sonata-arctica.wz.cz -sosdepotdebilan.com -specialfinanceoffers.com -speechfoodie.com -splendorsearch-a.akamaihd.net -sprttrack.com +shiksabd.com +shopvilleroyboch.com.ua +sisiynas.ru +site5.com +sizeplus.work +slimcdn.com +smartbalanceworld.com +snts.shell-pmr.ru +softomix.net +solution4u.com +songoo.wz.cz +soserfis.com +sovetogorod.ru +space2019.top +spasswelt.net +spy-sts.com +spyfu.com +srdrvp.com +sstroy44.ru starpages.net -strangeduckfilms.com -sugvant.ru -supermama.top -supermodni.com.ua +streamin.to +strongholdsb.ru +superiends.org superstats.com -surffoundation.nl -survival.betteroffers.review -svnuppsalaorebro.se -syvertsen-da.no -tagil.zrus.org -tako3.com -taqywu51.soup.io -telefonsexi.com -testingads.pro -the-trader.net -thegolfclub.info -timecrimea.ru -titslove.yopoint.in -tizanidinemusclerelaxant.blogspot.com -tizanidinesideeffects.blogspot.com -toonfamilies.net +surintech.ac.th +szucs.ru +takethatad.com +tdsing.ru +techart24.com +telefonsexkostenlos.tk +tengohydar.tk +texbaza.by +themeforest.net +thenews-today.info +tilido.com +tizanidine4mgstreetvalue.blogspot.com +tizanidinedruginteractions.blogspot.com +tizanidinepain.blogspot.com +tizanidinerestlesslegsyndrome.blogspot.com +tizanidinestreetprice.blogspot.com +top-deal.com.pl +top10-online-games.com topcar-krasnodar.ru +torontoplumbinggroup.com torrentgamer.net -touchmods.fr -tourismvictoria.com -traffixer.com -traffpartners.com -trkdf.com -tubeoffline.com +tpu.ru +tracklead.net +tradedeals.biz +trafficmp.com +traffictrade.life +tramadolandtizanidine.blogspot.com +tripper.de +trk-4.net +trudogolik.net +truemfilelj.gq +tupper-shop.ru +twelvevisionspartyofcolorado.com u17795.netangels.ru -unrealcommander.com -uptimebot.net -vancleefreplica.pw -vbabule.net -velen.io -ventelnos.com -viagraneggrx.com -vip2ch.com -vipromoffers.com -virus-respirators.com -vitanail.ru -vizitki.net -vltai.com -vodaodessa.com -vroze.com -vzlomfb.com -wapsite.me -watsonrealtycorp.com -webads.co.nz -weblibrary.win -weburlopener.com -weddingdresses.xyz -wejdz-tu.pl -wemedinc.com -wesharepics.site -westum.se -wfb.hatedriveapart.com -whatsappbot.flyland.ru -wholesalecheapjerseysfree.com -wineration.com -wonderfulflowers.biz -works.if.ua -worldhistory.biz -worst-sites.online -wosik-dach.service-for-web.de -wowcasinoonline.ooo -xboxster.ru -xdoza.com -xep.info -xn-----7kcabaipgeakzcss7bjdqdwpfnhv.xn--p1ai -xn----ctbbcjd3dbsehgi.xn--p1ai -xn--80aebbcbcdemfkhba4byaehoejh8dza3v.xn--p1ai -xn--80ak6aa92e.com -xn--d1abj0abs9d.in.ua -xn--d1aifoe0a9a.top -xxxtube69.com -yachts-cruise.info -ynymnwbm.bloger.index.hr -yubikk.info -zarabiaj-dzis.pl -zebradudka.com -zed21.net -zero1.it -zootoplist.com -zootravel.com -zskdla.site -zvooq.eu +u555u.info +uggsale.online +univerfiles.com +uptime-delta.net +urlcut.ru +userequip.com +validdomain.xyz +valkiria-tk.ru +vbikse.com +video--production.com +video-chat.love +video-hollywood.ru +videochat.ph +vkgaleria.com +voluumtrk.com +voprosotvet24.ru +vriel.batcave.net +watchmygf.net +wcb.su +weboptimizes.com +website-analytics.online +welck.octopis.com +welcomeauto.ru +weprik.ru +westvilletowingservices.co.za +whatsupinfoley.com +whengirlsgowild.com +wholesalejerseychinashop.com +wladimirpayen.com +woman-h.ru +word-vorlagen.net +workle.website +wrona.it +wtsindia.in +www.labves.ru +xblognetwork.com +xfluro.com +xn----9sbebi2bvzr7h.xn--p1ai +xxxsiterips.xyz +yellowfootprints.com +yellowproxy.net +yoopsie.com +youjizz.com +youthreaders.com +yuweng.info +zajm-pod-zalog-nedvizhimosti.ru +zambini.ru +zeroredirect7.com +zetmaster.ru +zlatnajesen.com +zoolubimets.ru +zscalertwo.net +zzbroya.com.ua diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index e63329342..a494beade 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -123any.com -51unlim.ru -5kstore.com -actionnooz.com -adminshop.com -adult3dgames.com -afmuseum.com -ahmedabadwebs.com -aihelen.net -aktobe.xkaz.org -amazingpic.net -anaseracresar.tk -angkortours.vn -apartmentratings.com -appleid-verification.com -artdestshop.eu -autodo.info -automatic-seo.com -avek.ru -aversis.net +100dollars-seo.com +2000k.ru +4kplayer.pl +7zap.com +8lufu.com +999webdesign.com +aasoldes.fr +acortarurl.es +adserver-e7.com +adultfullhd.com +advancedmassagebysara.com +albuteroli.com +algerianembassy.co.in +allbizne.co.ua +allfinweb.com +allwidewallpapers.com +alpinism.ru +altamayoztourism.com +alyeskaresort.com +amazon-adsystem.com +apibrowsesmartne-a.akamaihd.net +appearance-cool.com +applicationg29.com +ausmepa.org.au +australia-opening-times.com +aviav.co +aviav.ru avkzarabotok.com -bang-hotties.com -bannertesting.com -bashtime.ru -beauby.ru -beclean-nn.ru -bet365.com -biznesluxe.ru -bjgugu.net.ua -blog.koorg.ru -bluesman.nu -bonzbuddy.com -bum.com.ru -burger-imperia.com -burkesales.com -buydissertation.net -callstevens.com -cardsharp1.ru -carfax.com.ua -ceotrk.com -cheapjerseysfootballshop.com +babyguns.ru +beachpics.com +bestwrinklecreamnow.com +bibys.com +bitcoin-ua.top +bolegapakistan.com +book-bets.com +brandehk.dk +bravegirlsclub.com +brewdom.ru +buqyxa.rincian.info +businescoop.men +buyk.host.sk +buypanicdisorderpill.com +cakemediahq.com.au +candytiner.com +canrole.com +cheapestjerseys-wholesale.com chee-by.biz -chorus.terakeet.com -cityadspix.com -compliance-alex.top -compliance-jessica.top -craftinsta.ru -credit.co.ua -cryptoswap.biz -dangphoto.trade -davidovic.info -deanmoore.ie -devochki.top -dichvuvesinhhanoi.com -djihispano.com -djstools.com -doctormakarova.ru -dumpsmania24.com -dumuelave.xyz -e-c.al -easycommerce.cf -education-cz.ru -elektronischezigarettekaufen1.myblog.de -eropho.net -extlabs.io -ezaz.info -face.hostingx.eu -fantasticpixcool.com -faz99.com -feeriaclub.ru +chelyabinsk.xrus.org +chidporn.com +chuckguilford.com +cialischsrx.com +clickbank.net +cntravelre.com +cobaltpro.ru +compliance-nelson.top +concordexoticrentals.com +covadhosting.biz +cscwtalkto.site +daisye.top +daretodonate.co +davebestdeals.com +delta-line.men +digest-project.ru +discountbarbour.online +download-walpaperhd.blogspot.com +dprtb.com +electro-prom.com +elektrischezigaretten1.blogse.nl +eu-cookie-law.blogspot.com +european-torches.ru +event-tracking.com +evidencecleanergold.com +exchanges-bet.com +eyemagination.com +eyessurgery.ru +ezigarettenshop1.yolasite.com +ezigaretteshop2.vefblog.net fidelityfunding.com -fifa-coins.online -fighrofacciufreesig.ga -finejewelryshop.ru -finuse.com -florida-tourism.net -forexunion.net -forsex.info -fortevidyoze.net -foxjuegos.com -franklinfire.co -fsakhalin.ru -fungirlsgames.net -fyl.com.ru -galleryawesome.com -glogow.pl -goodwinmetals.co -gosmeb.ru -graphics8.info -hasfun.com -hawaiielectriclight.com -homelygarden.com -hoporno.com -igru-xbox.net -inome.com.ua -intim-uslugi.info -invitefashion.com -iweblist.info -jav-fetish.site -jerseychinabizwholesale.com +fivedwld.cf +foto-basa.com +fotosfotos.eu +free-video-tool.com +fxund.us +gamingspark.com +garciniaxt.us +gardene.ru +gateway.zscalerone.net +ge0ip.net +gok-kasten.net +gotcher.us +gribkovye-zabolevaniya.com +guitar-master.org +gwhwpxbw.bloger.index.hr +hentai-manga.porn +hermesbirkinhandbagoutlets.com +hermesbracelets.xyz +hitmuzik.ru +hkladys.com +hoverboard360.de +hulfingtonpost.com +hyhj.info +ico.re +ideibiznesa2015.ru +impisr.edunsk.ru +incest-ru.com +infazavr.ru +informatiecentro.be +infostatsvc.com +interior-stickers.ru +iradiology.ru +istmira.ru +itch.io +iwanttodeliver.com +izoll.ru +jackpotchances.com +jennyfire.ru +jerseyschinabizwholesale.us +jikoman.info johnnyhaley.top -joingames.org -journalhome.com -kevblog.top -keywordblocks.com -kinosed.net -kiwe-analytics.com -kountrylife.com -krasivoe-hd.com -kursy-ege.ru -lexaprogeneric.link -lietaer.com -lignofix.ua -llastbuy.ru -m.b00kmarks.com -m1media.net +jyrxd.com +kaac.ru +katushka.net +keyword-suggestions.com +keywordhouse.com +kinky-fetishes.com +kinocccp.net +kosmetyki.tm.pl +krasivoe-hd.net +krasula.pp.ua +ladsup.com +larutti.ru +limads.men +lob.com.ru +lockerz.com +love-baby.cz +lubetube.com +luxmagazine.cf m5home.ru -mabdoola.blogspot.com -macotool.com -magiadinamibia.blogspot.com -maglid.ru -mainlinehobby.net -makedo.ru -manipulyator-peterburg.ru -manualterap.roleforum.ru -map028.com -metaxalonevstizanidine.blogspot.com +menstennisforums.com +mezaruk.info mhtr.be -michaelkorsoutletstore.net -militarysale.pro -minecraft-rus.org -missclub.info +mmstat.com moinozhki.com -mojpregled.com moonci.ru -mp3downloadhq.com -music7s.me -nastydollars.com -nelc.edu.eg -netvouz.com -news-speaker.com -nfvsz.com -nhl09.ru -nodding-passion.tk -notebook-pro.ru -nrjmobile.fr -nw-servis.ru -oecnhs.info -offergroup.info -officedocuments.net -oksrv.com -onlyporno.ru -ons-add.men -onthemarch.co -paydayloanslocal.com -paydayonlinecom.com +mossmesi.com +moz.com +mydirtyhobby.com +nordstar.pro +novatech.vn +nudepatch.net +o-dachnik.ru +o333o.com +oakridgemo.com +oconto.ru +ostrovtaxi.ru +otbelivanie-zubov.com +paceform.com +pandarastore.top +pandroid.co pcimforum.com -phelissota.xyz -piski.top -piter.xrus.org -pizza-imperia.com -polybuild.ru -popup-fdm.xyz -presleycollectibles.com -privhosting.com -prom23.ru -qwesa.ru -rangjued.com -ranksonic.info -raschtextil.com.ua -realitykings.com -redhotfreebies.co.uk -relax.ru -responsive-test.net -reyel1985.webnode.fr -rootandroid.org -ruf777.com -rusoft-zone.ru -rybalka-opt.ru -sadaholding.com -scanmarine.info -scanner-donald.top -scanner-julianna.top -scanspyware.net -scenarii-1-sentyabrya.uroki.org.ua -scottbywater.com -se.bnt-team.com -semaltmedia.com -servethis.com -seryeznie-znakomstva.ru -sex-sex-sex5.com -sexxdate.net -sexy-pings.com -shijian.ac.cn -shopfishing.com.ua +perosan.com +picturesmania.com +pierrehardysale.online +piluli.info +plastjulgranar.se +pobeiranie.pl +pornofiljmi.com +pornoxxx.com.mx +pornozhara.com +porodasobak.net +prchecker.info +produkto.net +pronorm.fr +propranolol40mg.blogspot.com +qitt.ru +quickcashlimited.com +ranksignals.com +razorweb-a.akamaihd.net +realresultslist.com +redirlock.com +remontgruzovik.ru +rightenergysolutions.com.au +rozalli.com +rp9.ru +ruspdd.com +russia-today-video.ru +s-forum.biz +saitevpatorii.com +scansafe.net +seorankinglinks.com +sex-tracker.com +sexystrippe.info +sherlock.se +shodanhq.com +shtora66.ru simply.net -site-speed-check.site -siteonomy.com -sledstvie-veli.net -slimcdn.com -smartshoppymac.com -socas.pluto.ro -socialbuttons.xyz -sockshare.net -spitfiremusic.com -spravka130.ru -staticfs.host -sticken.co -stmassage.ru -stockspmb.info -superfish.com -superlist.biz -surcentro.com -symphonyintegratedhealthcare.com -takeprofitsystem.com -tarad.com -telefonsexkostenlos.tk -tgtclick.com -the-usa-games.blogspot.com -thefotosgratis.eu -tizanidinehcl4mgtablet.blogspot.com -tizanidinesleepaid.blogspot.com -tizanidinewithdrawal.blogspot.com -tonerbox.kz +sirpornogratis.xxx +slowmacfaster.trade +sobecjvuwa.com.ru +solmarket.by +storehouse.ua +stromerrealty.com +strongsignal-a.akamaihd.net +studiofmp.com +sweepstakes.rewardit.com +swiped.su +tedy.su +templates.radiodigital.co +thefarmergame.com +tizanidine4mguses.blogspot.com tonivedu.it -tpu.ru -trafaret74.ru -um-razum.ru -unionmarkt.de -usiad.net -ustion.ru -vereo.eu -vzlomtw.com -w-journal.ru -warningzscaler.heraeus.com -waycash.net -weclipart.com -whatistizanidine2mg.blogspot.com -wonderfulflowers.biz -woodyguthrie.se -wyniki-lista.pl -wzgyyq.com -xn----7sbahjd3btneuw1joc.xn--p1ai -xn--h1aakne2ba.xn--p1ai -xsfetish.org -youtuhe.com -zastenchivosti.net -zigarettenonlinekaufen2.bloog.pl -znakomstvaonlain.ru -zoogdiesney.com +tozup.com +trion.od.ua +tv-spoty.info +ultimateclassicrock.com +url2image.com +usacasino.com +usswrite.com +uzungil.com +v-doc.co +velen.io +ventelnos.com +video-hollywood.ru +vintontech.info +vizag.kharkov.ua +vkontarkte.com +vodaodessa.com +vrotike.ru +wcb.su +webjam.com +wedding-salon.net +wemedinc.com +womama.ru +wordkeyhelper.com +wufak.com +ww2awards.info +wwwadultcheck.com +xaijo.com +xchangetrak.com +xcombear.ru +xmnb.net +xn--80aaagvmjabrs1aoc9luc.xn--p1ai +xn--80aodinpgi.xn--p1ai +xn--b1addnj3cah.xn--p1ai +xxxnatelefon.ru +youporn-forum.ga +zarepta.com +zhuravlev.info +zoogdisany.com +zscalerone.net diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index bc1a013ac..54446da99 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.55 0; +10.1.1.54 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index 7f86f8155..95fd41db9 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -51154,3 +51154,1255 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2492 +### Updated: Fri Jul 30 14:08:15 UTC 2021 +### Bad Bot Count: 617 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ALittle Client +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: Leap +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: ubermetrics-technologies.com +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ + diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 7f86f8155..95fd41db9 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -51154,3 +51154,1255 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2492 +### Updated: Fri Jul 30 14:08:15 UTC 2021 +### Bad Bot Count: 617 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ALittle Client +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: Leap +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: ubermetrics-technologies.com +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ +