diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index fa989a5cd..c6c18ed42 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2457 +#### Version: V4.2021.07.2458 #### Bad Referrers Blocked: [7066](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 367746998..ca6f4dede 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2457 +#### Version: V4.2021.07.2458 #### Bad Referrers Blocked: [7066](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/README.md b/README.md index d9b66dcd7..68b1711eb 100755 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2021.07.2457 +#### Version: V4.2021.07.2458 #### Bad Referrers Blocked: [7066](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index 4ae633ea4..64c945876 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,7 +3,7 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2021.07.2457 +#### Version: V4.2021.07.2458 #### Bad Referrers Blocked: [7066](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [615](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index 6f23455c7..f5da48543 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2457 -### Updated: Mon Jul 5 12:40:46 UTC 2021 +### Version: V4.2021.07.2458 +### Updated: Mon Jul 5 12:52:42 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index 016872925..012b34b11 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2457 -### Updated: Mon Jul 5 12:40:47 UTC 2021 +### Version: V3.2021.07.2458 +### Updated: Mon Jul 5 12:52:43 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index 6f23455c7..f5da48543 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2457 -### Updated: Mon Jul 5 12:40:46 UTC 2021 +### Version: V4.2021.07.2458 +### Updated: Mon Jul 5 12:52:42 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf index 016872925..012b34b11 100644 --- a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2457 -### Updated: Mon Jul 5 12:40:47 UTC 2021 +### Version: V3.2021.07.2458 +### Updated: Mon Jul 5 12:52:43 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 4dc1e926a..fef092aba 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.56 1; +10.1.0.97 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 43a784cca..bce12f0ea 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.56 0; +10.1.0.97 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 016872925..012b34b11 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2457 -### Updated: Mon Jul 5 12:40:47 UTC 2021 +### Version: V3.2021.07.2458 +### Updated: Mon Jul 5 12:52:43 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 4dc1e926a..fef092aba 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.56 1; +10.1.0.97 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 43a784cca..bce12f0ea 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.56 0; +10.1.0.97 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index 016872925..012b34b11 100644 --- a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2457 -### Updated: Mon Jul 5 12:40:47 UTC 2021 +### Version: V3.2021.07.2458 +### Updated: Mon Jul 5 12:52:43 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 4dc1e926a..fef092aba 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.56 1; +10.1.0.97 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index 43a784cca..bce12f0ea 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.56 0; +10.1.0.97 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index 016872925..012b34b11 100644 --- a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2457 -### Updated: Mon Jul 5 12:40:47 UTC 2021 +### Version: V3.2021.07.2458 +### Updated: Mon Jul 5 12:52:43 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index 3274032dd..0af87fa88 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -2457 +2458 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index 016872925..012b34b11 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2457 -### Updated: Mon Jul 5 12:40:47 UTC 2021 +### Version: V3.2021.07.2458 +### Updated: Mon Jul 5 12:52:43 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index 6f23455c7..f5da48543 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2457 -### Updated: Mon Jul 5 12:40:46 UTC 2021 +### Version: V4.2021.07.2458 +### Updated: Mon Jul 5 12:52:42 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index f820c907c..1de51fdbd 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -52.173.134.38 0; +52.224.127.21 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index 6f23455c7..f5da48543 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2457 -### Updated: Mon Jul 5 12:40:46 UTC 2021 +### Version: V4.2021.07.2458 +### Updated: Mon Jul 5 12:52:42 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 4dc1e926a..fef092aba 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.56 1; +10.1.0.97 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index 43a784cca..bce12f0ea 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.56 0; +10.1.0.97 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index 6f23455c7..f5da48543 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2457 -### Updated: Mon Jul 5 12:40:46 UTC 2021 +### Version: V4.2021.07.2458 +### Updated: Mon Jul 5 12:52:42 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 4dc1e926a..fef092aba 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.56 1; +10.1.0.97 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index 43a784cca..bce12f0ea 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.56 0; +10.1.0.97 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index 6f23455c7..f5da48543 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2457 -### Updated: Mon Jul 5 12:40:46 UTC 2021 +### Version: V4.2021.07.2458 +### Updated: Mon Jul 5 12:52:42 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 4dc1e926a..fef092aba 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.56 1; +10.1.0.97 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index 43a784cca..bce12f0ea 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.56 0; +10.1.0.97 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index 6f23455c7..f5da48543 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2457 -### Updated: Mon Jul 5 12:40:46 UTC 2021 +### Version: V4.2021.07.2458 +### Updated: Mon Jul 5 12:52:42 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index f820c907c..1de51fdbd 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -52.173.134.38 0; +52.224.127.21 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index ce0586117..60b0d692a 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2457 -### Updated: Mon Jul 5 12:40:46 UTC 2021 +### Version: V4.2021.07.2458 +### Updated: Mon Jul 5 12:52:42 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index f820c907c..1de51fdbd 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -52.173.134.38 0; +52.224.127.21 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 14de823e8..eff6bc3d5 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2457 -### Updated: Mon Jul 5 12:40:46 UTC 2021 +### Version: V4.2021.07.2458 +### Updated: Mon Jul 5 12:52:42 UTC 2021 ### Bad Referrer Count: 7066 ### Bad Bot Count: 615 ################################################### diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index 4dc1e926a..fef092aba 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.56 1; +10.1.0.97 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 0f637e0c2..daf36a100 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -Alligator -BackWeb -Demon -Ecxi -Image Fetch -Iria -SBIder -Screaming -Sitevigil -TurnitinBot +CheTeam +DIIbot +FrontPage +IDBot +PictureFinder +Thumbor +WPScan +Who.is Bot +mediawords +webmeup-crawler diff --git a/dev-tools/test_units/random-bots-for-test.list b/dev-tools/test_units/random-bots-for-test.list index 66787f0da..c06830cc5 100644 --- a/dev-tools/test_units/random-bots-for-test.list +++ b/dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ 01h4x.com -AIBOT +404checker +404enemy ASPSeek -Aboundexbot +Abonti +AhrefsBot Aipbot -AllSubmitter +Alexibot AlphaBot Anarchy Ankit -Anthill +Apexoo +Aspiegel Asterias -AwarioRssBot -AwarioSmartBot -BDFetch -BackStreet -BackWeb -Backlink-Ceck +Attach +BLEXBot +BackDoorBot +BacklinkCrawler Badass +Bandit +BatchFTP +Bigfoot BlackWidow -Boardreader -BotALot +Bolt Brandprotect Brandwatch -Buck -Buddy -BuiltBotTough -CATExplorador -CODE87 -CensysInspect -CheTeam -Chlooe -Cliqzbot +BuiltWith +BuzzSumo +Calculon Cogentbot Copier -CopyRightCheck -Cosmos -CrazyWebCrawler +Copyscape +Craftbot CrunchBot -Curious Custo -CyotekWebCopy DBLBot -DSearch +DIIbot +DTS Agent DatabaseDriverMysqli -Discoverybot -Dispatch -DittoSpyder -DomainStatsBot +Devil +Digincore +DigitalPebble +Discobot +DomainAppender +DomainCrawler Dotbot -Download Wonder -Drip -EMail Wolf -EasyDL -EirGrabber -EroCrawler -Evil -Express WebPictures -Extractor +ECCP/1.0 +EMail Siphon +Ecxi +ExtLinksBot ExtractorPro EyeNetIE -FHscan +FDM +Fimap FlashGet -Freeuploader +Foobot FrontPage -Fuzz -FyberSpider -Fyrebot -G-i-g-a-b-o-t GermCrawler GetRight -Gigablast -Gigabot +GetWeb +Getintent Go!Zilla -GoZilla +Gotit +GrabNet +Grabber Grafula -GrapeshotCrawler HMView -HTTP::Lite Haansoft -Hloader -Humanlinks -HybridBot +HaosouSpider +Harvest +Havij IDBot -IRLbot -Iblog Id-search -Image Sucker InfoTekies Intelliseek -JOC Web Spider +Internet Ninja +Iskanie +JamesBOT Jbrofuzz -Jetty -Joomla -Kozmosbot -LeechGet -Lftp +JustView +Kenjin Spider +Kinza +Lanshanbot +Larbin +LeechFTP Libwhisker -Lightspeedsystems -Likse +LinkScan LinkWalker -Linkbot Linkdexbot +LinksManager LinqiaMetadataDownloaderBot +LinqiaScrapeBot Lipperhey -Ltx71 +Lipperhey Spider MIDown tool -MJ12bot +MIIxpc MSFrontPage -MTRobot -Magnet +Majestic-SEO +MarkWatch Mass Downloader +Masscan Mata Hari Mb2345Browser MeanPath Bot -Mediatoolkitbot +MegaIndex.ru Metauri +MicroMessenger Microsoft Data Access -Morfeus Fucking Scanner -Msrabot +Microsoft URL Control +Moblie Safari +Mojolicious +Mr.4x3 +Musobot NICErsPRO -NPbot -Name Intelligence +Nameprotect Navroad NearSite Needle -Nessus NetLyzer -NetMechanic +NetSpider NetZIP -Nibbler +Netcraft +Nettrack Nikto -NimbleCrawler +Nimbostratus Ninja Nuclei OPPO A33 -Octopus Offline Explorer Offline Navigator -OpenLinkProfiler +OnCrawl Openfind +Openvas OrangeBot OrangeSpider -OutfoxBot -PECL::HTTP -Page Analyzer +OutclicksBot +PageAnalyzer +PageScorer PageThing.com -Petalbot -Pi-Monster +Pandalytics +Panscient +Pavuk +PeoplePal Picscout -Piepmatz -Pixray -ProPowerBot -Pu_iN -PxBroker -PyCurl -RankActiveLinkBot -RankingBot -Rankivabot +Picsearch +PleaseCrawl +ProWebWalker +Probethenet +QueryN Metasearch +Quick-Crawler +RankurBot RealDownload -Recorder -RepoMonkey Ripper -Rogerbot -SEOkicks -SEOstats +RocketCrawler SISTRIX -SalesIntelligent +SMTBot ScanAlert -Scrapy -Screaming -ScreenerBot +Scanbot +ScoutJet +ScrepyBot Searchestate +SearchmetricsBot +Semrush SentiBot -SeoSiteCheckup -SeobilityBot -Site Sucker -SiteSucker -Sitebeam -Sitevigil -SmartDownload +Shodan +Siphon +SiteExplorer Snapbot Snoopy -SocialRankIOBot Sociscraper +Sogou web spider +SpaceBison Spanner -Spbot -Spinn3r -SputnikBot -Sqlmap -Sqlworm -Steeler -Sucuri -Surfbot +Sucker +Suzuran +Szukacz T0PHackTeam -T8Abot -Telesoft -Titan +Teleport +TeleportPro +Telesphoreo +Telesphorep +Toata Toweyabot Tracemyfile -Turnitin +Trendictionbot Typhoeus -URLy.Warning Upflow -VeriCiteCrawler +VB Project +Vagabondo +VidibleScraper Voil -WBSearchBot +Voltron +WASALive-Bot +WEBDAV +WISENutbot +WPScan WWW-Collector-E -WWW-Mechanize -Web Collage -Web Enhancer +WWW::Mechanize +Wallpapers/3.0 +WallpapersHD Web Fuck -Web Sauger +Web Pix +Web Sucker WebAuto -WebBandit WebFetch -WebFuck -WebImageCollector -WebSauger +WebReaper +WebStripper +WebSucker +WebWhacker Webalta +WebmasterWorldForumBot Webshag Website Quester -Whack -Whatweb -WinHTTrack -Wotbox -Zauba -ZmEu +WebsiteExtractor +Webster +Who.is Bot +Wonderbot +Woobot +Xaldon WebSpider +YoudaoBot +Zade +ZoomBot ZoominfoBot -archive.org_bot -arquivo-web-crawler +ZyBorg +autoemailspider backlink-check -cognitiveseo com.plumanalytics -crawl.sogou.com -crawler.feedback crawler4j demandbase-bot -facebookscraper -gopher -ips-agent +domainsproject.org +eCatch lwp-request -meanpathbot +lwp-trivial +netEstate NE Crawler +oBot page scorer pcBrowser -scan.lol +plumanalytics +polaris version +probe-image-size +seobility +seoscanners serpstatbot +sexsearcher sitechecker.pro -tAkeOut +spyfu +sysscan +voyagerx.com webmeup-crawler -x09Mozilla +xpymep1.exe diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index a99d6751e..0a9a00047 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -01h4x.com 360Spider +404checker ADmantX -Aboundexbot -Acunetix -AfD-Verbotsverfahren +Aboundex +AhrefsBot AiHitBot Alexibot -Anarchy -Anthill +AllSubmitter +Alligator Apexoo Aspiegel Attach -AwarioSmartBot -Backlink-Ceck -Bandit -Barkrowler -BatchFTP -Bigfoot -Bitacle -BlackWidow -Blow -Boardreader +BDFetch +BLEXBot +BacklinkCrawler +Badass +BetaBot +BlowFish Bolt +BotALot +Brandprotect Buck Buddy +BuiltBotTough BuiltWith -BunnySlippers +BuzzSumo +CATExplorador +CCBot +CODE87 +CSHttp Calculon -CheTeam ChinaClaw -Chlooe -Claritybot -Cogentbot -Collector -Copier +Copyscape Craftbot -CrunchBot -DIIbot +CrazyWebCrawler +Crescent +Curious +CyotekWebCopy +DSearch DTS Agent -Digincore -Disco +DataCha0s +DigitalPebble +Discobot DittoSpyder -DnyzBot -DomCopBot DomainCrawler -DomainStatsBot +DomainSigmaCrawler Domains Project -Dotbot -EMail Wolf -EirGrabber -EroCrawler -Exabot +Download Wonder +Evil Express WebPictures +ExtLinksBot Extractor ExtractorPro +Extreme Picture Finder EyeNetIE Ezooms FemtosearchBot -Fimap Firefox/7.0 -Fuzz +Foobot +Freeuploader +G-i-g-a-b-o-t GalaxyBot -Getintent +GermCrawler +GetRight Gigabot -GrabNet -HEADMasterSEO -HTMLparser +Go!Zilla +Go-Ahead-Got-It +Grabber +HMView HTTP::Lite HTTrack -HaosouSpider -Harvest +Haansoft Humanlinks -IDBTE4M +HybridBot IDBot +IlseBot +Image Fetch Image Sucker Intelliseek -IstellaBot +InterGET JOC Web Spider -JamesBOT -JennyBot +Jbrofuzz +JikeSpider Jorgee +Keyword Density +Kinza Kozmosbot -LNSpiderguy -LWP::Simple -LeechFTP +Lanshanbot +Larbin LeechGet Lftp -LibWeb +Libwhisker LieBaoFast -Likse -LinkScan -Linkbot +Lightspeedsystems +Linkdexbot LinkextractorPro LinksManager -LinqiaRSSBot +LinqiaMetadataDownloaderBot LinqiaScrapeBot Lipperhey Lipperhey Spider Litemage_walker Lmspider -MFC_Tear_Sample MIIxpc +MJ12bot +MQQBrowser Mag-Net -Mail.RU_Bot -Majestic SEO +Magnet Majestic-SEO MarkMonitor -Masscan Mata Hari -MeanPath Bot -Microsoft Data Access -Microsoft URL Control -Mister PiX +Mb2345Browser +Meanpathbot +Mediatoolkitbot +Metauri Moblie Safari -Msrabot -Name Intelligence -Nameprotect -Needle +Mojeek +Mojolicious +NICErsPRO +Navroad +Nessus NetAnts -NetMechanic -NetZIP -Nettrack +NetLyzer Netvibes -Nikto -Nimbostratus -Offline Navigator +Nibbler +Ninja +Nutch OpenLinkProfiler +Openvas OrangeBot OrangeSpider OutclicksBot PECL::HTTP -PageAnalyzer PageGrabber PageScorer -PageThing.com -Pandalytics -Papa Foto +Panscient Pavuk -Picscout -Piepmatz -Pixray -PleaseCrawl +Pi-Monster +Pimonster +Pockey ProPowerBot -ProWebWalker -Probethenet -Psbot +Pu_iN Pump -PxBroker +Quick-Crawler +RSSingBot RankActiveLinkBot -Rankivabot +RankFlex +RankingBot RankurBot -Re-re -Reaper -RebelMouse +RealDownload Recorder -RedesScrapy -SEOkicks -SEOlyticsCrawler -SEOprofiler -SEOstats -SMTBot +Ripper +Rogerbot +SEOkicks-Robot +SISTRIX +ScanAlert +Scanbot +Scrapy Screaming -ScreenerBot -Searchestate -SearchmetricsBot -Semrush -SentiBot -Shodan +SeoSiteCheckup Site Sucker SiteExplorer -Siteimprove -Sitevigil +SiteSnagger +SiteSucker +Sitebeam SmartDownload Snoopy -SocialRankIOBot Sociscraper +Sogou web spider Sottopop -Spammen -SpankBot -Sqworm -Sucker -SurveyBot +Sqlmap +Stripper +SuperBot +Surfbot +Swiftbot T0PHackTeam -Teleport +T8Abot Telesoft -Telesphorep The Intraformant -TheNomad -Toata +TightTwatBot Tracemyfile Trendictionbot -Turingos -Turnitin +TurnitinBot +Twice +UnisterBot +Upflow +V-BOT VCI +Vacuum Vagabondo -VelenPublicWebCrawler -Virusdie +VeriCiteCrawler +VoidEYE WBSearchBot WEBDAV -WPScan -WallpapersHD +WISENutbot +WWW-Collector-E +WWW-Mechanize +WWW::Mechanize +Wallpapers/3.0 WeSEE Web Enhancer Web Pix -Web Sauger -Web Sucker -WebAuto -WebEnhancer -WebFuck -WebLeacher +WebCollage +WebFetch +WebImageCollector WebPix -WebReaper +WebSauger WebStripper -WebmasterWorldForumBot +WebSucker +WebsiteExtractor +WebsiteQuester +Webster +Whack Whacker +Whatweb Who.is Bot -Widow Wonderbot -Woobot Wotbox -Xaldon_WebSpider -Zauba +Xaldon WebSpider +Xenu +YoudaoBot +Zade Zeus -ZoomBot -arquivo.pt +Zitebot +ZumBot +ZyBorg +autoemailspider backlink-check +coccocbot-web +crawler.feedback +crawler4j demandbase-bot +domainsproject.org eCatch -evc-batch +facebookscraper gopher -internetVista monitor +instabid +ips-agent lwp-trivial -magpie-crawler -muhstik-scan -netEstate NE Crawler -plumanalytics +page scorer +pcBrowser polaris version -probe-image-size -s1z.ru satoristudio.net -scalaj-http sexsearcher -sogouspider +sitechecker.pro spyfu -tAkeOut +sysscan trendiction.com x09Mozilla -xpymep1.exe -zauba.io +x22Mozilla diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 2708a6cb9..529ff3146 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -aliexpresscashback.club -apiwebwebgetcom-a.akamaihd.net -buydissertation.net -fres-news.com -movies-in-theaters.net -moyaterapiya.ru -myplaycity.com -sendearnings.com -vapeface.club -yoshkarola.zrus.org +ameblo.top +bambi.ck.ua +ccbill.com +easyshoppermac.com +jerseywholesaleelitestore.com +lizace.com +sabaapress.com +stjamesschool.info +tizanidinesleepaid.blogspot.com +xn--80ab4aa2g.xn--p1ai diff --git a/dev-tools/test_units/random-referrers-for-test.list b/dev-tools/test_units/random-referrers-for-test.list index f413011ab..16243461a 100644 --- a/dev-tools/test_units/random-referrers-for-test.list +++ b/dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -007angels.com -100dollars-seo.com -18ps.ru -214jaluwobapef.cf -3dracergames.com -4pp13.com +00go.com +0c47f8422d3f.com +1millionusd.xyz 4webmasters.com -5kstore.com -8jn.dating-app.ru -9med.net -a14download.com -aa625d84f1587749c1ab011d6f269f7d64.com -abcdeg.xyz -acgs.tk -actulite.com -addtoadd.men -adman.se -adrenalin-stk.ru +4webmasters.org +51unlim.ru +6128786.com +abbanreddy.com +abcguru.xyz +acc.eu.org +admanaerofoil.com +admeasures.com +adpremium.org +ads.tremorhub.com +adtiger.tk +advocatemsk.ru +adzerg.com agahinameh.com -ahmedabadwebs.com +agroeconom.kz aimayangzhi.com -albuteroli.com -allforminecraft.ru -allsilver925.co.il -alt-servis.ru -altamayoztourism.com +alevinefesleri.com +algerianembassy.co.in +alienwheels.de +alive-ua.com +all4wap.ru +alphacarolinas.com +alphavisions.net alvtank.se -alyeskaresort.com -amateurlivechat.org -amt-k.ru -anal-acrobats.com -angel1777.kz -aosexkontakte.net -apifasterlightin-a.akamaihd.net -apimountainbikei-a.akamaihd.net -apiportalmorecom-a.akamaihd.net -apiwoodensealcom-a.akamaihd.net -ardimobileinfo.ml -arendakvartir.kz -arkartex.ru -artdestshop.eu -asdfg.pro -asmxsatadriverin.aircus.com -asrv-a.akamaihd.net -atley.eu.pn -atovh.local-finders.com -automatic-seo.com -autovideobroadcast.com -bank.uz -bankcrediti.ru -bbsoldes.fr -beauty-b0x.pl -benchmarkcommunications.co.uk -best-seo-software.xyz -bestserials.com -bestsexyblog.com -biketank.ga -bingo8888.com -bizfly.info -blockety.co -blockworld.ru -bluejays-jerseys.us -bmwhighperformers.com -bocoarchives.org -boeuklad.com -bonanza-fortune.men -bonkers.name +am-se.com +anatomiy.com +android-style.com +android4fun.org +animebox.com.ua +anmysite.com +apisecretsaucebi-a.akamaihd.net +apptester.tk +aquarium-pro.ru +asophoto.com +atleticpharm.org +atmagroup.ru +autoplate.info +avtoarenda.by +avtocredit-legko.ru +ayanaspa.com +baditri.com +banki76.ru +beachdriveblog.com +beachfix.co +bensbargains.net +berrymall.ru +bestdraws.com +bestkfiledxd.cf +bestwebsiteawards.com +bgviagrachrx.com +bharatdefencekavach.com +bigpictures.club +bimmerpost.com +binomo.kz +biocn.dx.am +bitcoins-live.ru +bitporno.sx +bloggedporn.com +boostmyppc.com +braindaily.xyz +bravegirlsclub.com breget74.ru -bscodecs.com -buehne-fuer-menschenrechte.de -builtwith.com -bukleteg.xyz -butstrap.space -buy-forum.ru -buynorxx.com -calcularpagerank.com -carloans.com -cartierlove.xyz -cartierlovebraceletreplica.xyz -cbcseward.com -cdnnetwok.xyz -celejihad.info -centrumcoachingu.com +buscarfoto.com +buttons-for-website.com +buy-cheap-pills-order-online.com +bwlx.prepedu.cn +candiceloves.us +cannibalcheerleader.com +canrole.com +carapics.com +cardsdumps.com +cartierreplica.top +cartierring.xyz +cashkitten-a.akamaihd.net +castingbank.ru +catalogs-parts.com +cementaresearch.se +centrdebut.ru chatroulette.si -cheap-pills-norx.com -cheapjerseysa.com -chidporn.com -chlooe.com -christianlouboutinreplica.pw -cleansearch.net -coaching-netz.info -cocolyze.com -coldfilm.ru +cheapjerseysap.com +cherrypointplace.ca +christianlouboutinreplica.win +cialgenisrx.com +cibpenonptib.flu.cc +clubfashionista.com +coin-hive.com com-secure.download -compliance-fred.top -compliance-irvin.xyz +compliance-george.top +compliance-irvin.top compliance-josh.top -compliance-viktor.top -comsysnet.com -conboy.us -concealthyself.com -constantaservice.net -construmac.com.mx -coprofam.org -copyrightclaims.org -coslab.club -costablue.xyz -counterbot.com -courtshipgift.com -crazyprotocol.com -creams.makeforum.eu -credit.co.ua -crirussian.ru -csgo4.win +compliance-willy.top +contextualyield.com +cookie-law-enforcement-aa.xyz +cosmeticswomens-womensports.rhcloud.com +covid-schutzmasken.de +cpajunkies.com +crackzplanet.com +crest3d.ru +cubook.supernew.org +culturevie.info +cutt.us +cvety24.by +cymbaltaandtizanidine.blogspot.com dailyrank.net -data-mining.tk +dalavia.ru +daneshetabiat.com data1.scopich.com -dayibiao.com -ddospanel.com -ddsoldes.fr -decenttools.com -deda-moroza-zakaz.ru -dekoration.us +ddpills.com +de.zapmeta.com +dealighted.com +dear-diary.ru +delfin-aqua.com.ua depositfiles-porn.ga -detki-opt.ru -dfiles.me -dimfour.com -directrev.com -dojki-hd.com -donvito.unas.cz -downloads-whatsapp.com +dieta-personalna.pl +digital-video-processing.com +dikqlyremy.info +dikx.gdn +dirtpics.pw +doctissimo.fr +doctormakarova.ru +doktoronline.no +domainsphoto.com +domashneeruporno.com +domznaniy.ru +doreenblog.online +download-of-the-warez.blogspot.com +downvids.net +dpihatinh.gov.vn +drhomes.biz drpornogratisx.xxx -dstroy.su -dvervmoskvu.ru -dwomlink.info -e-avon.ru -education-cz.ru -edwinkonijn.com.au -efkt.jp -ege-russian.ru -eldiariodeguadalajara.com -electricwheelchairsarea.com -elektrischezigarette2.devhub.com -elektrozigaretten2.yn.lt -elkacentr.ru -eluxer.net -emerson-rus.ru -energy-ua.com +dvr.biz.ua +e-biznes.info +ebooknovel.club +editors.choice6912650.hulfingtonpost.com +ege-essay.ru +elektrischezigarette1.blog.pl +elite-sex-finders.com +emailaccountlogin.co +embedle.com +enbersoft.com +enews.tech +enpolis.ru +enthuse.computernetworksonline.com envaseslotusama.com eonpal.com -erotag.com -etur.ru -eu-cookie-law.info -expolicenciaslatam.co -extrabot.com -eyelike.com.ua -ezigarettekaufen1.hpage.com -ezigarettenshop2.postbit.com -f-online.de -fashion-mk.net -felizporno.com -findpics.pw -finstroy.net -flagcounter.me -floridamobilebillboards.com -foodgid.net -footballfarrago.com -forms-mtm.ru +epicbrogaming.com +erotik-kostenlos.net +essaypro.com +estibot.com +evogarage.com +exct.net +exmasters.com +factorynightclub.com +fakehandbags.xyz +familyphysician.ru +filmi-onlain.info +find1friend.com +fingerprintjs.com +fireads.men +fitness-video.net +flashahead.info +formseo.com +formulaantiuban.com foto-telok.net -fotoxxxru.com -free-laptop-reward.com +free-social-buttons.xyz +free411games.com +freefoto.ca +freshsuperbloop.com friendflnder.com -fugarif.ga -funnymama.com -fym.com.ru -g33.org -gamblingnerd.com -getyourimage.club -girlporn.ru -gk-atlant.info -glass-msk.ru -glastecfilms.com.my -glavprofit.ru -golaya.pw -gratis-sexkontakte.com -greatzip.com -gsasearchengineranker.xyz -hamptonoaks.ca -hanwei.us -happy.new.yeartwit.com -hard-porn.mobi -hatedriveapart.com -hdapp1008-a.akamaihd.net -hdfreeporno.net -headpress.ru -healgastro.com -healmytrauma.info -health-medical-portal.info -helpmymacfaster.trade -histats.com -hjaoopoa.top -hledejvshopech.cz -homedecoguide.info -hopeonthestreet.co.uk -hoverboard360.at -hscsscotland.com -husky-shop.cz -igrovyeavtomaty777.ru -ilmen.net -ilo134uloh.com -imagui.eu -imk.com.ua -incest-ru.com -indiakino.net -infobabki.ru -innoslicon.com -inome.com.ua -insta-add.pro -insurple.com -investpamm.ru -invitefashion.com -isistaylorporn.info -isitwp.com -itag.pw -itch.io -jacago.com -janettabridal.com -jav-idol.com -jav-way.site -jerseysbizwholesalecheap.com -juliadiets.com -juliaworld.net -jurajskie.info -kam-dom.ru -kasino-money.pw -kellyonline.xyz -kemerovo.zrus.org -keywordteam.net -kino2018.cc -klitimg.pw -klumba55.ru -knowyournextmove.com -kollekcioner.ru -komp-pomosch.ru +fuckbuddybestgilf.info +fullgirl.ru +fun-mobi.pl +fungamelands.com +funtoonez.com +galeon.com +gallictures.com +game300.ru +gamerextra.com +gate5.co.za +gateway.zscalerone.net +gay-file.com +gay-site.store +geckoandfly.com +getridofstretchmarks.org +given2.com +glicol.kz +goblacked.com +gosreg.amchs.ru +graphics8.info +growmyfunds.ca +gsasearchenginerankerdiscount.com +haikuware.com +handsandlegs.ru +hd-film.pl +highland-homes.com +historichometeam.com +hollywoodactress.info +hostingclub.lk +hotelcrocenzi.sm +hotshoppymac.com +housekuba.org +houtings.xyz +id-forex.com +ihc2015.info +ihtec2019.org +imagine-ex.co +imgata.com +imguramx.pw +iminent.com +impotentik.com +incolors.club +increasewwwtraffic.info +instasexyblog.com +interesnie-faktu.ru +ipchicken.com +iplogger.org +iptool.xyz +iwantmyfreecash.com +jamiembrown.com +jmat.cn +joessmogtestonly.com +journeydownthescale.info +jpcycles.com +jrpmakati.com +kabbalah-red-bracelets.com +kakablog.net +kareliatobacco.ru +kazrent.com +kenaba.su +kinky-fetishes.com +kinoflux.net +kinomaniatv.pl +kinotorka.ru +kinozapas.org +klejonka.info +kol-energo.ru konkursowo-24.pl -koronirealestate.gr -kvartiry-remont.ucoz.ru -kw21.org +konturkrasoty.ru +kostenloser-sex.com +kredit-blog.ru +kvartira-sutochno.com lacloop.info ladov.ru -lanasshop.ru -landoftracking.com -larger.io -larutti.ru -layola.biz.tc -le-clos-des-alouettes.com -lechenie-gemorroya.com -lescinq.com -letto.by -lexaprogeneric.link -libertybilisim.com -lightinghomes.net -linkbolic.com -liverpool.gsofootball.com -livesex-amateure.info -lollypopgaming.com +lietaer.com +liffa.ru +likesdesign.com +linkdebrideur.xyz +lion.snapmap.us +liran-locks.com +liver-chintai.org +liveu.infoteka.hu +loadingpages.me +locatellicorretor.com.br +locimge.pw longgreen.info louboutinbooties.xyz -louisvuittonoutletstore.net -m0r0zk0-krava.ru -macfix.life -macnewtech.com -malwareremovalcenter.com -map028.com -marcogrup.com -masthopehomes.com -matsdale.com -mecash.ru -meccadumps.net -medispainstitute.com.au -megagrabber.ru -megahdporno.net -meltwater.com -mesto-x.com -mlvc4zzw.space -mokrayakiska.com -montazhnic.ru -mosrif.ru -motherboard.vice.com -moviemail-online.co.uk -moyakuhnia.ru +louboutinreplica.xyz +love-baby.cz +luciddiagnostics.in +lutherstable.org +lyngdalhudterapi.no +mac-shield.com +machicon-akihabara.info +mactechinfo.info +magicalfind-a.akamaihd.net +magicdiet.gq +makis.nu +manifestyourmillion.com +mansparskats.com +matpre.top +meetingrainstorm.bid +meetlocalchicks.com +mejoresfotos.eu +mezaruk.info +microsoftportal.net +minet.club +mmstat.com +mncrftpcs.com +mobifunapp.weebly.com +mole.pluto.ro +mompussy.net +monitorwebsitespeed.com +mrbitsandbytes.com +muizre.ru mulberryoutletonlineeu.com -musirc.com -muz-shoes.ru -my-cash-bot.co -mydoctorok.ru -mylesosibirsk.ru -mymobilemoneypages.com -mypets.by -na15.ru -nakrutka.cc -narkologiya-peterburg.ru -narkologiya-voronezh.ru -nastroyke.net -nationalbreakdown.com -naturtreenspicerx.pw -net-profits.xyz -netpics.org -nextrent-crimea.ru -nordvpn.com -notaria-desalas.com -novosibirsk.xrus.org -o00.in -odoratus.net -oecnhs.info -offers.bycontext.com -offf.info -oklogistic.ru -olympescort.com -oneshotdate.com -online-x.ru -onlywoman.org -ororodnik.goodbb.ru -osagonline.ru -oyster-green.com +music7s.me +mvpicton.co.uk +najaden.no +nalogovyykodeks.ru +naperehresti.info +narkologiya-orel.ru +narutonaruto.ru +nash-krym.info +naturalbreakthroughsresearch.com +naturalpharm.com.ua +netanalytics.xyz +networkad.net +newtechspb.ru +nikitsyringedrivelg.pen.io +nw-servis.ru +obesidadealgarve.com +obiavo.kz +oneminutesite.it +onko-24.com +onlinemeetingnow.com paidonlinesites.com -paintingplanet.ru -parajumpersoutlet.online -partnerline.men -partnersafe.men -pastaleads.com -pattersonsweb.com -pensplan4u.com -perm.xrus.org -picmoonco.pw -pixell.club +paighambot.com +paradontozanet.ru +partner-high.men +partner-print.men +perfection-pleasure.ru +peugeot-club.org +phuketscreen.com +pics-group.com +picurams.pw +pila.pl +pivka.xyz pixelrz.com -plaff-go.ru -plastjulgranar.se -pokemongooo.ml -pokerniydom.ru -popugauka.ru -popup.matchmaker.com -pornhub-ru.com +platesauto.com +pogosh.com +pokemon-go-play.online +porndroids.com pornhubforum.tk -porno-chaman.info -pornodojd.ru +porno-video-chati.ru +pornoelita.info pornohub.me -portside.cc -positive2b.ru -potoideas.us -pourvous.info -preconnubial.usuby.site -printdirectforless.com -privat-girl.net -productarium.com -psa48.ru -pssucai.info +pornoklad.net +pornoklad.ru +pornophoto.xyz +pornoplen.com +pornzone.tv +porodasobak.net +porto.abuilder.net +privatefx-in.ru +privatov-zapisi.ru +prizefestival.mobi pulse33.ru -puteshestvennik.com -putevka24.ru -puzo2arbuza.ru -qualitymarketzone.com -queerspace.com +qitt.ru +qwertty.net +rabotaetvse.ru +raisedseo.com +ranksonic.info rapidgator-porn.ga rapidokbrain.com -rapidsites.pro -rarbg.to -recipedays.ru -redirect.trafficreceiver.club -registrationdomainsite.com -relayblog.com -rentehno.ru -reyel1985.webnode.fr -ring4rhino.com -risparmiocasa.bz.it -rrutw.com +razleton.com +realizmobi.com +realresultslist.com +reckonstat.info +remontbiz.ru +remote-dba.de +repeatlogo.co.uk +respublica-otel.ru +resultshub-a.akamaihd.net +ringporno.com +rocketchange.ru +rockma.se +roma-kukareku.livejournal.com +royal-investments.net rucrypt.com -salutmontreal.com -sanidumps.com -satoristudio.net -scalerite.co.za -scanner-alex.top -school-diplomat.ru -scopich.com -sdelai-prosto.ru -search.1and1.com -secretscook.ru -semprofile.com -seo-2-0.com -sevendays.com.ua -sexvporno.ru -sexy-screen-savers.com -shariki-zuma-lines.ru -sherlock.se -shop.xz618.com -shtaketniki.kz -shymkent.xkaz.org -sibvitr.ru -siha.de -simplepooltips.com -sindragosa.comxa.com -site5.com -sitechecker.pro -sitevaluation.org -smsactivator.ru -snegozaderzhatel.ru -snjatie-geroinovoy-lomki.ru -sobecjvuwa.com.ru -soft1.ru -softtor.com -souvenirua.com -spb-plitka.ru -stard.shop -stefanbakosab.se -stopnarco.ru -strana-solnca.ru -stroicol.net -superiends.org -superinterstitial.com -swagbucks.com -tarad.com -tastyfoodideas.com -taximytishi.ru -template-kid.com -thebluenoodle.com -thecoral.com.br -thedownloadfromwarez.blogspot.com -tizanidineandgabapentin.blogspot.com -tizanidinehcl2mgsideeffects.blogspot.com -tizanidinerecreationaluse.blogspot.com -tizanidinevscyclobenzaprine.blogspot.com +russia-tao.ru +rvi.biz +rybalka-opt.ru +sabizonline.com +sawin.beth.webd.pl +scanner-mary.top +scoopquest.com +scrapinghub.com +screentoolkit.com +se.bnt-team.com +seminarykansas904.ml +semrush.com +seoexperimenty.ru +seoheap.com +seoholding.com +servingnotice.com +sex-sex-sex5.com +sexkontakteao.info +sexkrasivo.net +shama-rc.net +sharebutton.net +sharebutton.to +shareyards.com +shemalegalls.blogporn.in +shijian.ac.cn +shitting.pro +shlyahten.ru +sideeffectsoftizanidine.blogspot.com +simple-image.com.ua +sirpornogratis.xxx +site-speed-check.site +snjack.info +snow.nvr163.com +soc-econom-problems.ru +softomix.com +sogimlecal.tk +solinf.co +speeddream.xyz +sports-supplements.us +spywarebegone.com +spywarenuker.com +spywarespy.com +standardchartered-forex.com +starwars.wikia.com +staynplay.net +stevemonsen.com +store-rx.com +stroiminsk.org +talant-factory.ru +tamada69.com +tattomedia.com +teenfuck.tv +templates.radiodigital.co +thelottosecrets.com +thewebsitetemplate.info +thruport.com +timer4web.com +tizanidinenarcotic.blogspot.com +tizanidinesideeffects.blogspot.com +toonfamilies.net top-l2.com -top-study.work -top10-way.com -topanasex.com -topdownloads.ru -torrent-newgames.com -totu.info -toy-shop.top -tradgardspartner.se -tuberkulezanet.ru -tupper-posuda.ru -tupper-shop.ru -turist-strani.ru -turizmus.us -twitlinks.com -ua.tc -uginekologa.com -uhdtv.website -unacittaconte.org -unrealcommander.com -uptime-gamma.net -urldelivery.com -usadacha.net -usswrite.com -vbabule.net -viagra-soft.ru -viagra.pp.ua -videochat.bz -videochaty.ru -villakohlanta.nu -vladhistory.com -voprosotvet24.ru -w-journal.ru -warning.or.kr -waycash.net -webinstantservice.com -weblo.com +top1-seo-service.com +topcar-krasnodar.ru +torrent-to-magnet.com +trackzapper.com +traffic100.com +trafficreceiver.club +trustaffs.com +tsstcorpcddvdwshbbdriverfb.aircus.com +tvory.predmety.in.ua +typer.one +ugguk.online +ukrup.com +umaseh.com +unblocksit.es +unimodemhalfduplefw.pen.io +unmaroll.ya.ru +uptime-delta.net +ural-buldozer.ru +usiad.net +ussearche.cf +valsalud.com +vasileostrovsky-rayon.ru +vellings.info +versaut.xxx-cam.webcam +vesnatehno.ru +video--production.com +video-camer.com +vipromoffers.com +virus-respirators.com +visa-pasport.ru +visionwell.com.cn +vizag.kharkov.ua +vkmusics.ru +vkonche.com +vkontaktemusic.ru +voloslove.ru +vykup-avto-krasnodar.ru +wardreapptokone.tk +webcamtalk.net +webix.biz +website-analytics.online website-speed-check.site +welck.octopis.com +wesharepics.info +westen-z.life +whereiskentoday.com whereverdesperate.gq -wholinkstome.com -wildnatureimages.com -witherrom55.eklablog.fr -womensplay.net -www.ehscloud.cn -www.get-free-traffic-now.com -xjlottery.com -xn------7cdbapdecfd4ak1bn0amjffj7afu3y.xn--p1ai -xn-----6kccaibs5cb8afhjrfmix2n.xn--p1ai -xn----7sbho2agebbhlivy.xn--p1ai -xn----itbeirbjbi7bc6bh2d.xn--p1ai -xn--b1ag5cfn.xn--p1ai -xtraffic.plus -yeartwit.com -yopoint.in -youdao.com -youradulthosting.com -yourothersite.com +wildworld.site +womama.ru +woman-tampon.ru +wtsindia.in +www.souvenirua.com +wygraj-skiny.win +x-true.info +xerox-douglas.cf +xev.ru +xlolitka.com +xn-----6kcaacnblni5c5bicdpcmficy.xn--p1ai +xn--80aaafbn2bc2ahdfrfkln6l.xn--p1ai +xn--lifehacer-1rb.com +xnxxandxvideos.com +xolodremont.ru +xpresscare.ru +xtremeeagles.net +xvideospornoru.com +xwatt.ru +yellocloud.be yuarra.pluto.ro -zajm-zalog-krasnodar.ru -zakazvzloma.com -zeg-distribution.com +z-master.ru +zazagames.org +zero1.it zeroredirect10.com -zhorapankratov7.blogspot.com -zigarettenonl.canalblog.com -zojirushi-products.ru -zrus.org -zskdla.site -zvetki.ru +znakomstvaonlain.ru +znaturaloriginal.com +zvuker.net +zzlgxh.com diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index 608e498cc..bfff4620b 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -1314dh.com -1pamm.ru -24videos.tv -4webmasters.com -9med.net -a14download.com -aa14ab57a3339c4064bd9ae6fad7495b5f.com -aadroid.net -acunetix-referrer.com -adprotect.net -adsfresh.men -advocatemsk.ru -aerogo.com -afftrack001.com -algarveglobal.com -alphaforum.ru -anaseracresar.tk -android-vsem.org -anniemation.com -asrv-a.akamaihd.net -aviav.co -balitouroffice.com -basedecor.ru -bcmp.org +404.mn +aa625d84f1587749c1ab011d6f269f7d64.com +aa9bd78f328a6a41279d0fad0a88df1901.com +abclauncher.com +adobereader-free.ru +agapovdg.ru +agreda.pluto.ro +aimayangzhi.com +alldubai.biz +anal-acrobats.hol.es +angigreene.com +apartmentbay.ru +apessay.com +apibetweenlinesn-a.akamaihd.net +apiitsthirteende-a.akamaihd.net +apimossnetinfo-a.akamaihd.net +apiwoodensealcom-a.akamaihd.net +asdfz.pro +avtochehli.by +avtocredit-legko.ru beauty-bracelet.com -belstaffstore.online -big-boards.info -biznesluxe.ru -blackcurranthumidifiers.site -blagovest-med.ru -bodybuilding-shop.biz -bonanza-fortune.men -bosefux.esy.es -bosman.pluto.ro -brandbucket.com -business-made-fun.com -buy-cheap-pills-order-online.com -buy-meds24.com -buyviagraa.blogspot.com -candycrushshop.com -cascadelink.org -centraletermice.us -cheapjerseysfootballshop.com -cheapmoncler.win -checkhit.com -chelnytruck.ru -chelyabinsk.xrus.org -chimiver.info -cinemaenergy-hd.ru -cjs.com.ru -cleanersoft.com -cobaltpro.ru -compliance-alex.xyz -compliance-donald.xyz -computernetworksonline.com -counter.bloke.com -czeshop.info -danschawbel.com -data.vtc.pw -ddlmega.net -destinationrealestate.com -divisioncore.com -docs4all.com -domain-submit.info -domik-derevne.ru -e-c.al -e-kwiaciarz.pl -earl-brown.info -ekobata.ru -election.interferencer.ru -elektrozigaretten1.tumblr.com -elitedollars.com -emediate.eu -englishdictionaryfree.com -erot.co -errorfixing.space -exonline.info -eyes-on-you.ga +bestmarriages.com +bestpornuha.com +blackplanet.com +bobinoz.com +bocoarchives.org +boleznikogi.com +brokergid.ru +capture-room.com +casite-513049.cloudaccess.net +cheapjerseysap.com +chocoslim.pro +cjmc.info +clickcash.com +clips.ua.ac.be +compliance-irvin.top +compliance-mary.top +compliance-olga.top +converse.ddsoldes.fr +customchocolate.business-for-home.com +cyse.tk +delayreferat.ru +dfiles.me +diamond-necklace.info +dieswaene.com +doska-vsem.ru +drivotracker.com +e-commerce-seo1.com +easyshoppermac.com +editors.choice6912650.hulfingtonpost.com +elektrischezigarette1.blog.pl +elektrozigarettekaufen2.blogse.nl +encodable.com +erotik0049.com +eupornstar.info +ezigarettekaufen1.hpage.com +ezigarettekaufen2.yolasite.com +ezrvrentals.com +f0815.de +fashion-stickers.ru fatmaelgarny.com -fetlifeblog.com -floridamobilebillboards.com -forensicpsychiatry.ru -franch.info -freejabs.com -fullgirl.ru -funnel.co.za -gambarkatabaru.com -game-mmorpg.net -game-top.su -gateway.zscalertwo.net -gcup.ru -get-seo-domain.com -gg-arena.ru -golden-catalog.pro -gomusix.com -goodwinmetals.co -graphics8.info -guildebzh.info -gwagka.com -homedecoguide.info -hosting-tracker.com -ihc2015.info -imagecoolpub.com -isotoner.com -karadene.com -keywords-monitoring-your-success.com -khadastoafarde.tk -kidd.reunionwatch.com -kimcurlrvsms.com +fillmewithhappiness.com +findthe.pet +food.dtu.dk +forums.d2jsp.org +freetangodownload.com +frivgame250.com +frvo.alptandem.ru +fuck-paid-share-buttons.xyz +fun-mobi.pl +funnypica.com +gallictures.com +gdebestkupit.ru +gemgrab-a.akamaihd.net +generic-pills-online.com +getlaid-xxxhookupdirect.com +gheus.altervista.org +ghostvisitor.com +gk170.ru +goodprotein.ru +gopro-online.info +gosmeb.ru +gsasearchengineranker.space +gsmlab.pl +highstairs-a.akamaihd.net +homedo.fabpage.com +hostingclub.lk +hotelcrocenzi.sm +ideashome.id +iflycapetown.co.za +imabase.com +images-free.net +infobanks.ru +investmac.com +investpamm.ru +izismile.com +jerseyschinabizwholesale.us +jovencitas.gratis +kaac.ru +kalb.ru +kaz.kz +keywordhut.com +keywordspay.com kinky-fetishes.com -kino-fun.ru -koversite.info -kw21.org -larger.io -lerporn.info -letmacwork.world -libertybilisim.com -likesuccess.com -lincolntheatre.com -localmatchbook.com -louboutinbooties.xyz -low-format.ru -luchshie-topcasino.ru -magento-crew.net -markjaybeefractal.com -maslenka.kz -max-p.men -mebeldekor.com.ua -megavolt.net.ua -meteocast.net -micasainvest.com -milblueprint.com -missis.top -mlvc4zzw.space -mole.pluto.ro -more-letom.ru -moroccosurfadventures.com -mototsikl.org -mrmoneymustache.com -myonigroup.com -mywallpaper.top -new-apps.ru -nicovideo.jp -niki-mlt.ru -no-fuel.org -nonameread45.live -northfacestore.online -notfastfood.ru -o333o.com -osagonline.ru -pb-dv.ru -pc-services.ru +kino-filmi.com +kinobaks.com +kinocccp.net +kinotorka.ru +kirov.zrus.org +lacloop.info +languagecode.com +lapolis.it +laudit.ru +lesbian.xyz +lifebyleese.com +limtu.ifmo.ru +littlecity.ch +live-sexchat.ru +lonely-mature.com +lookingglassemb.com +mabdoola.blogspot.com +machicon-akihabara.info +mackeeper-center.club +manimpotence.com +maximilitary.ru +med-zdorovie.com.ua +medi-fitt.hu +merryhouse.co.uk +miniads.ca +missclub.info +mobplayer.net +modabutik.ru +money-for-placing-articles.com +morlat.altervista.org +mp3downloadhq.com +myprintscreen.com +myshopmatemac.com +mzdish.site +nalogovyy-kodeks.ru +nardulan.com +netvouz.com +nhl17coins.exblog.jp +nvssf.com +nw-servis.ru +oddamzadarmo.eu +of-ireland.info +offer.wpsecurity.website +onlinemegax.com +opinionreelle.com +partner-pop.men +partnersafe.men pcimforum.com -perfection-pleasure.ru -phelissota.xyz -picscout.com -pila.pl -pills24h.com -portadd.men -potolokelekor.ru -prizesbook.online -prizestohandle.club -productarium.com -prohoster.info -proprostatit.com -qwarckoine.com -reward-survey.net -rockingclicks.com -roll123.com -rp9.ru -rusprostitute.com -s-iwantyou.com -scanner-alex.top -search.alot.com -seccioncontrabajo.com -sei80.com -serpstat.com -sexad.net -sfd-chess.ru -sinel.info -sisiynas.ru -siteexpress.co.il -sitevaluation.org -sklad-24.ru -slim.sellany.ru -slomm.ru +pictures-and-images.net +pinwallpaper.top +pio.polytopesexempt.com +pipki.r.acdnpro.com +pistonclasico.com +pix24x7.com +pl-vouchers.com +pornoelita.info +pozvonim.com +pshare.biz +ptr.ruvds.com +push-ad.com +quickcashlimited.com +quizzitch.net +radiodigital.co +realizmobi.com +reklama-i-rabota.ru +remedyotc.com +replicalouboutin.xyz +resant.ru +retreatia.com +sbdl.no +sc-specialhost.com +scanner-donald.top +scenarii-1-sentyabrya.uroki.org.ua +scenicmissouri.us +se.bnt-team.com +seansonline24.pl +seeingmeerkat.com +seo-prof1.xyz +servingnotice.com +sexpornotales.net +shakhtar-doneck.ru +share-buttons-for-free.com +sideeffectsoftizanidine.blogspot.com +smstraf.ru snip.to -social-search.me -sotkal.lark.ru -soundfrost.org -splendorsearch-a.akamaihd.net -standardchartered-forex.com -steebook.com -stockquotes.wooeb.com -studiokamyk.com.pl -supernew.org -svensk-poesi.com -symbaloo.com -szamponrevita.pl -tako3.com -tamada69.com -tastyfoodideas.com -td-33.ru -technopellet.gr -tedy.su -thecoolimages.net -tiens2010.ru -tizanidinecapsules.blogspot.com -tizanidinedruginteractions.blogspot.com -topclickguru.com -topshef.ru +soheavyblog.com +songplanet.ru +speeddream.xyz +spinnerco.ca +sprttrack.com +sptslmtrafms.com +srgwebmail.nl +stair.registrydomainservices.com +stard.shop +stauga.altervista.org +steame.ru +stockspmb.info +stroicol.net +strv.se +sukirgenk.dvrlists.com +t-bygg.com +teencastingporn.com +tetracsaudi.com +textads.men +thegamerznetwork.com +tizanidine6mg.blogspot.com +tizanidineandhydrocodone.blogspot.com +tizanidinepills.blogspot.com +tizanidinesleepaid.blogspot.com +tmearegion26.com torture.ml -traffic100.com -ultimateclassicrock.com +traffique.net +twittrading.com +uhdtv.website +ukkala.xyz +ukrtextbook.com +ukrtvir.com.ua +ultimatesetnewfreeallsoftupgradesystems.pw upstore.me -uroffer.link -vapeface.club -varikozdok.ru -veerotech.com -videositename.com -videtubs.pl -vip-dom.in -vladimir.xrus.org -vzglyadriv.kg -vzlomsn.org -wapsite.me -webcamtalk.net -webs.com -website-stealer.nufaq.com -wejdz-tu.pl -westsextube.com -wholesalejerseychinashop.com -wikes.20fr.com -winterclassichockeyjerseys.com -woman-orgasm.ru -wowas31.ucoz.ru -www.arenda-yeisk.ru -wygraj-skiny.win -x5market.ru -xjlottery.com -xn-----6kcaacnblni5c5bicdpcmficy.xn--p1ai -xn--80abgj3a5acid6ghs.top -yogamatsexpert.com -yopoint.in -youbrainboost.asia -za-fun-offer.com -zakazfutbolki.com -zhongwenlink.com -zoomovies.org -zootravel.com +uzporno.mobi +videochat.ph +vinylvault.co.uk +vipromoffers.com +vipsexfinders.com +vzubah.com +wemarketing.se +wieseversa.no +witherrom55.eklablog.fr +wonderfulflowers.biz +worldmusicfests.com +wsgames.ru +ww2awards.info +xgames-04.com +xn----itbkqkfiq.xn--p1ai +xn--80aodinpgi.xn--p1ai +xn--h1ahbi.com.ua +xnxx-n.com +yadro.ru +ypmuseum.ru +zajm-zalog-krasnodar.ru +zastenchivosti.net +zdesformula.ru +zeikopay.com +zhorapankratov7.blogspot.com +zixizop.net.ru +zkjovpdgxivg.ga diff --git a/dev-tools/test_units/ratelimittest-beta.txt b/dev-tools/test_units/ratelimittest-beta.txt index b2122ba99..3656720f2 100644 --- a/dev-tools/test_units/ratelimittest-beta.txt +++ b/dev-tools/test_units/ratelimittest-beta.txt @@ -1,11 +1,21 @@ - + + + +Welcome to NginX + + +
+

Welcome to the NginX Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + 503 Service Temporarily Unavailable

503 Service Temporarily Unavailable

nginx/1.18.0 (Ubuntu)
-ml> + 503 Service Temporarily Unavailable

503 Service Temporarily Unavailable

diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index 43a784cca..bce12f0ea 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.1.56 0; +10.1.0.97 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index 084464466..cca350596 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -8734,3 +8734,1251 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2458 +### Updated: Mon Jul 5 12:52:44 UTC 2021 +### Bad Bot Count: 615 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: OPPO A33 +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ + diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 084464466..cca350596 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -8734,3 +8734,1251 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2458 +### Updated: Mon Jul 5 12:52:44 UTC 2021 +### Bad Bot Count: 615 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: OPPO A33 +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ +