From 21a2ecd4c0ea96af2abc6d15d30cb65ff5bb881b Mon Sep 17 00:00:00 2001 From: mitchellkrogza Date: Sat, 31 Jul 2021 14:11:35 +0000 Subject: [PATCH] V4.2021.07.2494 [ci skip] --- AUTO-CONFIGURATION.md | 2 +- MANUAL-CONFIGURATION.md | 2 +- README.md | 2 +- _google_analytics_ghost_spam/README.md | 2 +- .../etc/nginx/conf.d/globalblacklist.conf | 4 +- conf.d/globalblacklist-testing-version.conf | 4 +- conf.d/globalblacklist.conf | 4 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- dev-tools/buildnumber | 2 +- dev-tools/globalblacklist-testing.template | 4 +- dev-tools/globalblacklist.template | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/blacklist-ips.conf | 2 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- .../bots.d/whitelist-ips.conf | 2 +- .../conf.d/globalblacklist.conf | 4 +- dev-tools/test_units/blacklist-ips.conf | 2 +- .../random-bots-for-test-quick.list | 18 +- .../test_units/random-bots-for-test.list | 296 ++-- .../random-bots-for-whitelist-test.list | 302 ++-- .../random-referrers-for-test-quick.list | 20 +- .../test_units/random-referrers-for-test.list | 920 ++++++------ .../random-referrers-for-whitelist-test.list | 484 +++---- dev-tools/test_units/ratelimittest-master.txt | 14 +- dev-tools/test_units/whitelist-ips.conf | 2 +- dev-tools/tmp/robots.txt | 1252 +++++++++++++++++ robots.txt/robots.txt | 1252 +++++++++++++++++ 46 files changed, 3588 insertions(+), 1074 deletions(-) diff --git a/AUTO-CONFIGURATION.md b/AUTO-CONFIGURATION.md index a6b363e6a..56440ccbf 100755 --- a/AUTO-CONFIGURATION.md +++ b/AUTO-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2493 +#### Version: V4.2021.07.2494 #### Bad Referrers Blocked: [7072](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [617](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/MANUAL-CONFIGURATION.md b/MANUAL-CONFIGURATION.md index 38ff749ff..22f83b0fc 100755 --- a/MANUAL-CONFIGURATION.md +++ b/MANUAL-CONFIGURATION.md @@ -4,7 +4,7 @@ ### PLEASE READ CONFIGURATION INSTRUCTIONS BELOW THOROUGHLY :exclamation: _______________ -#### Version: V4.2021.07.2493 +#### Version: V4.2021.07.2494 #### Bad Referrers Blocked: [7072](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [617](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/README.md b/README.md index 85f6596eb..e5d00d75d 100755 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ ##### The Ultimate Nginx Bad Bot, User-Agent, Spam Referrer Blocker, Adware, Malware and Ransomware Blocker, Clickjacking Blocker, Click Re-Directing Blocker, SEO Companies and Bad IP Blocker with Anti DDOS System, Nginx Rate Limiting and Wordpress Theme Detector Blocking. Stop and Block all kinds of bad internet traffic even Fake Googlebots from ever reaching your web sites. [PLEASE SEE: Definition of Bad Bots](#define-bad-bots) _______________ -#### Version: V4.2021.07.2493 +#### Version: V4.2021.07.2494 #### Bad Referrers Blocked: [7072](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [617](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_google_analytics_ghost_spam/README.md b/_google_analytics_ghost_spam/README.md index d1c67cddd..eb10e0a86 100644 --- a/_google_analytics_ghost_spam/README.md +++ b/_google_analytics_ghost_spam/README.md @@ -3,7 +3,7 @@ # EASY CONFIGURATION INSTRUCTIONS FOR STOPPING GOOGLE ANALYTICS "GHOST" SPAM _______________ -#### Version: V4.2021.07.2493 +#### Version: V4.2021.07.2494 #### Bad Referrers Blocked: [7072](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-referrers.list) #### Bad User-Agents (Bots) Blocked: [617](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/bad-user-agents.list) #### Fake Googlebots Blocked: [217](https://raw.githubusercontent.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker/master/_generator_lists/fake-googlebots.list) diff --git a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf index d67295e93..99535291b 100755 --- a/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf +++ b/_sample_config_files/Engintron_for_cPanel_WHM_Configuration_Example/etc/nginx/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2493 -### Updated: Sat Jul 31 10:20:36 UTC 2021 +### Version: V4.2021.07.2494 +### Updated: Sat Jul 31 14:05:40 UTC 2021 ### Bad Referrer Count: 7072 ### Bad Bot Count: 617 ################################################### diff --git a/conf.d/globalblacklist-testing-version.conf b/conf.d/globalblacklist-testing-version.conf index a220a4cb0..abc392cea 100644 --- a/conf.d/globalblacklist-testing-version.conf +++ b/conf.d/globalblacklist-testing-version.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2493 -### Updated: Sat Jul 31 10:20:37 UTC 2021 +### Version: V3.2021.07.2494 +### Updated: Sat Jul 31 14:05:41 UTC 2021 ### Bad Referrer Count: 7072 ### Bad Bot Count: 617 ################################################### diff --git a/conf.d/globalblacklist.conf b/conf.d/globalblacklist.conf index d67295e93..99535291b 100644 --- a/conf.d/globalblacklist.conf +++ b/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2493 -### Updated: Sat Jul 31 10:20:36 UTC 2021 +### Version: V4.2021.07.2494 +### Updated: Sat Jul 31 14:05:40 UTC 2021 ### Bad Referrer Count: 7072 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf index a220a4cb0..abc392cea 100644 --- a/dev-tools/beta_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2493 -### Updated: Sat Jul 31 10:20:37 UTC 2021 +### Version: V3.2021.07.2494 +### Updated: Sat Jul 31 14:05:41 UTC 2021 ### Bad Referrer Count: 7072 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 198ed19bd..b8418242c 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.198 1; +10.1.0.132 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index d47ae2e04..170f5965e 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.198 0; +10.1.0.132 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf index a220a4cb0..abc392cea 100644 --- a/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2493 -### Updated: Sat Jul 31 10:20:37 UTC 2021 +### Version: V3.2021.07.2494 +### Updated: Sat Jul 31 14:05:41 UTC 2021 ### Bad Referrer Count: 7072 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 198ed19bd..b8418242c 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.198 1; +10.1.0.132 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf index d47ae2e04..170f5965e 100644 --- a/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.198 0; +10.1.0.132 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf index a220a4cb0..abc392cea 100644 --- a/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2493 -### Updated: Sat Jul 31 10:20:37 UTC 2021 +### Version: V3.2021.07.2494 +### Updated: Sat Jul 31 14:05:41 UTC 2021 ### Bad Referrer Count: 7072 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf index 198ed19bd..b8418242c 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.198 1; +10.1.0.132 1; 127.0.0.1 1; diff --git a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf index d47ae2e04..170f5965e 100644 --- a/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/beta_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.198 0; +10.1.0.132 0; 127.0.0.1 0; diff --git a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf index a220a4cb0..abc392cea 100644 --- a/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/beta_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2493 -### Updated: Sat Jul 31 10:20:37 UTC 2021 +### Version: V3.2021.07.2494 +### Updated: Sat Jul 31 14:05:41 UTC 2021 ### Bad Referrer Count: 7072 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/buildnumber b/dev-tools/buildnumber index e7ff0dbf8..e2e75c06b 100644 --- a/dev-tools/buildnumber +++ b/dev-tools/buildnumber @@ -1 +1 @@ -2493 +2494 diff --git a/dev-tools/globalblacklist-testing.template b/dev-tools/globalblacklist-testing.template index a220a4cb0..abc392cea 100644 --- a/dev-tools/globalblacklist-testing.template +++ b/dev-tools/globalblacklist-testing.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V3.2021.07.2493 -### Updated: Sat Jul 31 10:20:37 UTC 2021 +### Version: V3.2021.07.2494 +### Updated: Sat Jul 31 14:05:41 UTC 2021 ### Bad Referrer Count: 7072 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/globalblacklist.template b/dev-tools/globalblacklist.template index d67295e93..99535291b 100755 --- a/dev-tools/globalblacklist.template +++ b/dev-tools/globalblacklist.template @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2493 -### Updated: Sat Jul 31 10:20:36 UTC 2021 +### Version: V4.2021.07.2494 +### Updated: Sat Jul 31 14:05:40 UTC 2021 ### Bad Referrer Count: 7072 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf index 6bacfb50f..b9e5c7a7f 100644 --- a/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -168.61.177.33 0; +40.78.128.207 0; diff --git a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf index d67295e93..99535291b 100644 --- a/dev-tools/test1_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2493 -### Updated: Sat Jul 31 10:20:36 UTC 2021 +### Version: V4.2021.07.2494 +### Updated: Sat Jul 31 14:05:40 UTC 2021 ### Bad Referrer Count: 7072 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf index 198ed19bd..b8418242c 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.198 1; +10.1.0.132 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf index d47ae2e04..170f5965e 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.198 0; +10.1.0.132 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf index d67295e93..99535291b 100644 --- a/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ip_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2493 -### Updated: Sat Jul 31 10:20:36 UTC 2021 +### Version: V4.2021.07.2494 +### Updated: Sat Jul 31 14:05:40 UTC 2021 ### Bad Referrer Count: 7072 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf index 198ed19bd..b8418242c 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.198 1; +10.1.0.132 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf index d47ae2e04..170f5965e 100644 --- a/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_ratelimiting/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.198 0; +10.1.0.132 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf index d67295e93..99535291b 100644 --- a/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_ratelimiting/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2493 -### Updated: Sat Jul 31 10:20:36 UTC 2021 +### Version: V4.2021.07.2494 +### Updated: Sat Jul 31 14:05:40 UTC 2021 ### Bad Referrer Count: 7072 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf index 198ed19bd..b8418242c 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.198 1; +10.1.0.132 1; 127.0.0.1 1; diff --git a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf index d47ae2e04..170f5965e 100644 --- a/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf +++ b/dev-tools/test1_conf_files_whitelist/bots.d/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.198 0; +10.1.0.132 0; 127.0.0.1 0; diff --git a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf index d67295e93..99535291b 100644 --- a/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf +++ b/dev-tools/test1_conf_files_whitelist/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2493 -### Updated: Sat Jul 31 10:20:36 UTC 2021 +### Version: V4.2021.07.2494 +### Updated: Sat Jul 31 14:05:40 UTC 2021 ### Bad Referrer Count: 7072 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf index 6bacfb50f..b9e5c7a7f 100644 --- a/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test2_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -168.61.177.33 0; +40.78.128.207 0; diff --git a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf index 3ab010a37..c74921811 100644 --- a/dev-tools/test2_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test2_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2493 -### Updated: Sat Jul 31 10:20:36 UTC 2021 +### Version: V4.2021.07.2494 +### Updated: Sat Jul 31 14:05:40 UTC 2021 ### Bad Referrer Count: 7072 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf index 6bacfb50f..b9e5c7a7f 100644 --- a/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf +++ b/dev-tools/test3_conf_files/bots.d/whitelist-ips.conf @@ -45,4 +45,4 @@ # - Whitelisting IP's and RANGES here ONLY affects the IP blocking functions. # - This file will NOT allow your own IP to bypass bad User-Agent or Referrer String checks. # - To bypass everything for a certain IP see notes in blockbots.conf on SUPER WHITELIST -168.61.177.33 0; +40.78.128.207 0; diff --git a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf index 3f3530f0d..b3eb1ec98 100644 --- a/dev-tools/test3_conf_files/conf.d/globalblacklist.conf +++ b/dev-tools/test3_conf_files/conf.d/globalblacklist.conf @@ -4,8 +4,8 @@ ### VERSION INFORMATION # ################################################### -### Version: V4.2021.07.2493 -### Updated: Sat Jul 31 10:20:36 UTC 2021 +### Version: V4.2021.07.2494 +### Updated: Sat Jul 31 14:05:40 UTC 2021 ### Bad Referrer Count: 7072 ### Bad Bot Count: 617 ################################################### diff --git a/dev-tools/test_units/blacklist-ips.conf b/dev-tools/test_units/blacklist-ips.conf index 198ed19bd..b8418242c 100644 --- a/dev-tools/test_units/blacklist-ips.conf +++ b/dev-tools/test_units/blacklist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.198 1; +10.1.0.132 1; 127.0.0.1 1; diff --git a/dev-tools/test_units/random-bots-for-test-quick.list b/dev-tools/test_units/random-bots-for-test-quick.list index 0a4647aba..416464b7e 100644 --- a/dev-tools/test_units/random-bots-for-test-quick.list +++ b/dev-tools/test_units/random-bots-for-test-quick.list @@ -1,10 +1,10 @@ -BBBike -Brandwatch -GrapeshotCrawler -MegaIndex.ru -Psbot +BackWeb +Bolt +Cocolyzebot +Custo RankingBot -Semrush -TeleportPro -Telesphoreo -Trendictionbot +WWW-Collector-E +WWW-Mechanize +Web Collage +WebFuck +polaris version diff --git a/dev-tools/test_units/random-bots-for-test.list b/dev-tools/test_units/random-bots-for-test.list index 0e5c2ae6d..5169f04e0 100644 --- a/dev-tools/test_units/random-bots-for-test.list +++ b/dev-tools/test_units/random-bots-for-test.list @@ -1,250 +1,250 @@ +01h4x.com 360Spider -404checker -80legs +404enemy +ADmantX ALittle Client -Abonti -Acunetix +ASPSeek AhrefsBot AiHitBot -AlphaBot -Anarchie -Anarchy99 -Ankit -Aspiegel -AwarioSmartBot -BBBike -BDCbot +Aipbot +AllSubmitter +Apexoo +Attach BDFetch BackWeb -BacklinkCrawler +Backlink-Ceck Bandit -Barkrowler -BatchFTP -Black Hole +Battleztar Bazinga +BetaBot +Blow BlowFish Boardreader -BotALot -BuiltBotTough -Bullseye -BunnySlippers +Bolt +Brandwatch BuzzSumo CATExplorador -CODE87 CSHttp Calculon -CazoodleBot +CensysInspect CheeseBot -CherryPicker +ChinaClaw +Cliqzbot Cloud mapping Cocolyzebot -Collector -Cosmos +Copyscape Craftbot CrazyWebCrawler -Crescent -Curious -CyotekWebCopy -DBLBot DIIbot +DTS Agent DataCha0s -DatabaseDriverMysqli -Deusu +DigitalPebble Disco DittoSpyder -DomainCrawler -Domains Project -Download Wonder +DnyzBot +ECCP/1.0 EMail Siphon -EasyDL -EroCrawler -Express WebPictures -ExtLinksBot +Ebingbong +Extractor FHscan +FemtosearchBot FlashGet -Foobot +Flunky +Freeuploader Fuzz -Fyrebot -G-i-g-a-b-o-t +FyberSpider GT::WWW GalaxyBot +GermCrawler GetRight -GetWeb -Getintent -GrabNet +Gigablast +Go-Ahead-Got-It +Gotit Grabber -Grafula -GrapeshotCrawler -HEADMasterSEO -HTTrack -Heritrix -Hloader +HTMLparser +HTTP::Lite +Havij Humanlinks IDBTE4M +IRLbot +Iblog Id-search +IlseBot Image Fetch -Image Sucker -IndeedBot -InfoTekies -InterGET -Iskanie +InfoNaviRobot +Intelliseek +Internet Ninja +IstellaBot JOC Web Spider -JennyBot -JetCar -JikeSpider +Joomla JustView +Jyxobot Kenjin Spider -Kinza -Lanshanbot -LeechGet +Keyword Density +Kozmosbot +LWP::Simple LexiBot -Lftp -LibWeb -LieBaoFast -Lightspeedsystems +Libwhisker Likse -LinqiaMetadataDownloaderBot -LinqiaRSSBot +LinkScan +Linkbot +Linkdexbot +LinkextractorPro +LinkpadBot +Lipperhey Lipperhey Spider -Litemage_walker Lmspider +Ltx71 MFC_Tear_Sample -MQQBrowser -MSFrontPage -MarkMonitor -MeanPath Bot -Meanpathbot -MicroMessenger -Microsoft Data Access +MIIxpc +MJ12bot +MTRobot +Magnet +Majestic SEO +Majestic-SEO +Majestic12 +Mass Downloader +Masscan +Mb2345Browser +Mediatoolkitbot +Microsoft URL Control +Mister PiX +Mojeek Morfeus Fucking Scanner -Mozlila -Mr.4x3 Msrabot NICErsPRO -Name Intelligence -Nameprotect -NearSite -Nessus -Net Vampire -NetAnts -NetSpider +Needle +Netcraft NextGenSearchBot +Niki-bot +Nikto NimbleCrawler +Nimbostratus +Ninja +Nmap Nuclei Nutch +Offline Explorer OpenVAS -Openfind Openvas -OutfoxBot +OrangeSpider PECL::HTTP +PHPCrawl POE-Component-Client-HTTP -PageGrabber -PageThing.com -Pavuk +PageScorer +PeoplePal Petalbot +Picscout Picsearch +PictureFinder +Pimonster Pixray PleaseCrawl -ProPowerBot -Probethenet -Psbot +ProWebWalker Pu_iN -Pump PxBroker PyCurl +QueryN Metasearch +Quick-Crawler RSSingBot -RankActiveLinkBot +RankFlex +RankingBot RankingBot2 ReGet -Reaper RedesScrapy RepoMonkey +Ripper RocketCrawler -SEOlyticsCrawler +Rogerbot SEOprofiler -Scanbot +SISTRIX +ScanAlert Scrapy -SentiBot +ScreenerBot +SearchmetricsBot SeoSiteCheckup -Seomoz +SeobilityBot +Siphon Site Sucker SiteSnagger +Siteimprove Sitevigil -SmartDownload -Snapbot -Snoopy Sociscraper +Sosospider +SpaceBison Spammen -Spanner Spbot SputnikBot Sqlmap -Sqworm -SuperBot +Sqlworm +Steeler +Stripper SuperHTTP -Surfbot -Suzuran Szukacz T0PHackTeam -Teleport -Telesoft -The Intraformant -TheNomad -Toata -Trendictionbot -Turnitin -TurnitinBot -Twice +TeleportPro +Telesphorep +TightTwatBot +Tracemyfile +True_Robot +Turingos +TwengaBot +Typhoeus +URLy.Warning UnisterBot V-BOT VB Project -Virusdie -VoidEYE -WASALive-Bot -WBSearchBot +Vagabondo +VeriCiteCrawler WEBDAV WISENutbot -WallpapersHD +WWW-Mechanize +Wallpapers/3.0 WeSEE Web Collage Web Fetch -WebBandit -WebGo IS -WebPix +Web Fuck +Web Sauger +WebAuto WebReaper WebStripper WebZIP +WebmasterWorldForumBot Whack -Who.is Bot +Whacker Widow WinHTTrack -Wonderbot -Xaldon_WebSpider -Xenu -Zade +WiseGuys Robot +Wotbox +Wprecon +YoudaoBot +Zauba +Zermelo +Zeus Zitebot -ZmEu -archive.org_bot +ZumBot +arquivo.pt +autoemailspider +backlink-check cognitiveseo -com.plumanalytics -crawl.sogou.com -eCatch -evc-batch +crawler4j +demandbase-bot +domainsproject.org +facebookscraper +heritrix internetVista monitor +ips-agent isitwp.com +lwp-request magpie-crawler -mediawords -probe-image-size -ripz -s1z.ru -satoristudio.net -scan.lol -serpstatbot +muhstik-scan +netEstate NE Crawler +oBot +pcBrowser +polaris version sexsearcher -sp_auditbot -spyfu -sysscan +tAkeOut +trendiction.com ubermetrics-technologies.com -voyagerx.com webmeup-crawler -x09Mozilla -xpymep1.exe +zauba.io +zgrab diff --git a/dev-tools/test_units/random-bots-for-whitelist-test.list b/dev-tools/test_units/random-bots-for-whitelist-test.list index 71b8421ed..177ef8326 100644 --- a/dev-tools/test_units/random-bots-for-whitelist-test.list +++ b/dev-tools/test_units/random-bots-for-whitelist-test.list @@ -1,250 +1,250 @@ -404checker -ALittle Client +360Spider +80legs Aboundex -Aboundexbot +Acunetix AfD-Verbotsverfahren AhrefsBot -Aipbot Alligator -AlphaBot Anarchie +Anarchy Anarchy99 -Aspiegel +Ankit +Apexoo Asterias Attach -AwarioRssBot -AwarioSmartBot -BackDoorBot +BDFetch BackStreet -Barkrowler -Blackboard -BotALot +Badass +BatchFTP +Battleztar Bazinga +Bitacle +Black Hole +BlowFish +Boardreader +Bolt Brandprotect -Buddy +Brandwatch +Buck BuiltBotTough -Bullseye -BunnySlippers -CATExplorador -CazoodleBot -Cegbfeieh -CensysInspect -CheeseBot +CSHttp +CherryPicker Chlooe -Cloud mapping +Claritybot Cocolyzebot Collector +CopyRightCheck Cosmos -CrunchBot -DSearch -DatabaseDriverMysqli +Craftbot +Curious +Custo +DIIbot +DTS Agent Devil Digincore -Dirbuster -Discoverybot +Disco +Discobot Dispatch -DomainCrawler -DomainSigmaCrawler -Domains Project -Dotbot -Download Wonder -Ebingbong +DittoSpyder +DomCopBot +DomainStatsBot +Dragonfly +ECCP/1.0 +EMail Siphon +EMail Wolf Ecxi +EirGrabber EroCrawler Exabot Express WebPictures +ExtLinksBot Extractor Extreme Picture Finder -EyeNetIE FDM Fimap FlashGet -FyberSpider -GT::WWW +Foobot +Freeuploader +FrontPage +G-i-g-a-b-o-t +GalaxyBot GermCrawler -Getintent +GetWeb Gigablast Gigabot -Go-Ahead-Got-It -GoZilla -Grafula +GrabNet GrapeFX -HTMLparser +HMView HTTP::Lite -HTTrack -Haansoft +HaosouSpider +Havij Heritrix +Hloader Humanlinks -IDBTE4M -IDBot -Iblog +IRLbot Id-search -Image Fetch -InfoNaviRobot InterGET -Internet Ninja -Iria +InternetSeer Iskanie -JOC Web Spider -Jbrofuzz -Jorgee -Keyword Density -Lanshanbot -LeechGet -Lftp +IstellaBot +JamesBOT +JennyBot +Jetty +Jyxobot +Kozmosbot +LNSpiderguy +LeechFTP +LibWeb Libwhisker -Lightspeedsystems +LieBaoFast Likse LinkScan -LinkWalker -Linkdexbot -LinkextractorPro -LinkpadBot -LinksManager -LinqiaScrapeBot +Lipperhey +Lipperhey Spider Litemage_walker +Lmspider Ltx71 -MIIxpc +MFC_Tear_Sample MJ12bot -MSIECrawler +MSFrontPage Magnet +Majestic12 +MarkMonitor +MarkWatch Mass Downloader -Mb2345Browser +Masscan +Mata Hari MeanPath Bot Meanpathbot -MicroMessenger +Mediatoolkitbot Microsoft Data Access -Mojeek -Mojolicious -Morfeus Fucking Scanner -Mr.4x3 -NPbot -Navroad -NearSite -Needle -Nessus -NetAnts -NetZIP -Netcraft +Microsoft URL Control +Mister PiX +Moblie Safari +Msrabot +Musobot +NICErsPRO +NetLyzer +NetSpider Nibbler -Nikto -Nutch -Octopus +Niki-bot +Ninja +Nmap Offline Explorer -OnCrawl -OpenVAS -Openfind -OrangeBot +Offline Navigator +OrangeSpider +PECL::HTTP POE-Component-Client-HTTP Page Analyzer PageScorer -Pandalytics -Panscient -Picsearch +PageThing.com +Petalbot +Pi-Monster Pimonster PleaseCrawl Pockey ProPowerBot Probethenet -Psbot -PxBroker -PyCurl -QueryN Metasearch -RSSingBot +Pu_iN +RankActive RankActiveLinkBot -RankingBot2 -RankurBot -Re-re -ReGet +Rankivabot RealDownload -SEOlyticsCrawler +Reaper +Recorder +RepoMonkey +Ripper SEOprofiler SEOstats -SMTBot -ScreenerBot +SISTRIX +SalesIntelligent +Scanbot +Scrapy Searchestate -SearchmetricsBot +Semrush SemrushBot -SentiBot -SeobilityBot +Seomoz Shodan -Siphon -SiteSucker -SlySearch +SiteCheckerBotCrawler +Sitebeam +Sitevigil +Snake Sociscraper -Sogou web spider +Sosospider +Sottopop SpaceBison +Spammen Spanner SputnikBot -Sqlworm +Sqlmap +Sqworm Sucuri -SuperHTTP -Swiftbot +SurveyBot Szukacz +T8Abot TeleportPro -The Intraformant -TightTwatBot -Titan -Toata +Telesphoreo +Toweyabot +Tracemyfile Trendictionbot +True_Robot +Turingos Turnitin TurnitinBot +Twice URLy Warning -UnisterBot -Upflow +V-BOT VCI -Vacuum -Vagabondo -VelenPublicWebCrawler -VidibleScraper -Virusdie -VoidEYE +Voltron +WASALive-Bot +WBSearchBot +WISENutbot +WWW-Mechanize +WWWOFFLE Wallpapers/3.0 -WeSEE Web Auto +Web Collage Web Enhancer +Web Fuck Web Pix Web Sucker -WebAuto WebBandit -WebGo IS -WebPix -WebsiteExtractor +WebCopier +WebFetch +WebFuck +WebReaper +WebSauger +WebSucker +WebWhacker +Webalta +Webshag WebsiteQuester -WinHTTrack -WiseGuys Robot +Wonderbot +Woobot +Wotbox +Xaldon WebSpider Xenu -YoudaoBot -Zauba -Zermelo -Zeus +Zade Zitebot -arquivo.pt -autoemailspider -check1.exe +ZumBot +ZyBorg +coccocbot-web +com.plumanalytics crawl.sogou.com crawler.feedback -demandbase-bot +facebookscraper heritrix -internetVista monitor -ips-agent lwp-request -meanpathbot -mediawords +magpie-crawler muhstik-scan +netEstate NE Crawler oBot -pcBrowser plumanalytics polaris version ripz -seostar +s1z.ru +scalaj-http sexsearcher -sitechecker.pro -sogouspider -spyfu -tAkeOut +siteripz trendiction.de voyagerx.com -x22Mozilla -xpymep1.exe zauba.io -zgrab diff --git a/dev-tools/test_units/random-referrers-for-test-quick.list b/dev-tools/test_units/random-referrers-for-test-quick.list index 51b708e5c..f41117eb0 100644 --- a/dev-tools/test_units/random-referrers-for-test-quick.list +++ b/dev-tools/test_units/random-referrers-for-test-quick.list @@ -1,10 +1,10 @@ -apibrowsesmartne-a.akamaihd.net -autogrep.ru -braindaily.xyz -bristolhotel.com -imguramx.pw -of-ireland.info -piratecams.com -rossmark.ru -shop.xz618.com -trahvid.com +4kpics.rocks +abusora.com +fifa-coins.online +gorabagrata.ru +headpress.ru +mall.uk +perm.xrus.org +seryeznie-znakomstva.ru +sexreliz.com +shoppytoolmac.com diff --git a/dev-tools/test_units/random-referrers-for-test.list b/dev-tools/test_units/random-referrers-for-test.list index f0d67defc..bc6c6e97c 100644 --- a/dev-tools/test_units/random-referrers-for-test.list +++ b/dev-tools/test_units/random-referrers-for-test.list @@ -1,500 +1,500 @@ -00go.com -1-free-share-buttons.com -100searchengines.com -2728fb936f0.com -2daytrendingnews.com -3rbseyes.com -40cg.com -55wmz.ru -7a2rnuey1tw9ar.ru -7makemoneyonline.com -98oi.ru -acads.net -actionnooz.com -addtoadd.men -adsland.men -advokateg.xyz -afora.ru -agardomains.com -akita.kz -alert-hgd.xyz -alfa-medosmotr.ru -alldubai.biz -allnews.md +28n2gl3wfyb0.ru +2ads.co.uk +2rich4bitches.com +300richmond.co.nz +34.gs +4inn.ru +4istoshop.com +5i2.net +5u.com +888.com +8gold.com +a1security.com.ua +aa14ab57a3339c4064bd9ae6fad7495b5f.com +aa8780bb28a1de4eb5bff33c28a218a930.com +aac52006ec82a24e08b665f4db2b5013f7.com +aaronabel.com +adloads.com +adtiger.tk +adultfullhd.com +adultwebhosting.info +alienwheels.de +allcredits.su +alldezservice.kz +allnews24.in aloofly.com -amateurmatch.com -amt-k.ru -anaksma.info -anniemation.com -anonymizeme.pro -anonymous-redirect.com -apccargo.com -apifasterlightin-a.akamaihd.net -apimossnetinfo-a.akamaihd.net -apiportalmorecom-a.akamaihd.net -arate.ru -ardimobileinfo.ml -arenanews.com.ua -arenda-yeisk.ru -asmxsatadriverin.aircus.com +alphaforum.ru +amazingpic.net +ameblo.jp +apioutoboxnet-a.akamaihd.net +approved.su +appsecurityr.com +arendovalka.xyz +arewater.com +artefakct.com +arvut.org +asdfz.pro asseenontv.ru -atout-energie-69.com -audiofree.ru -autobudpostach.club +atdedinotuho.tk +autonew.biz autorn.ru -aviav.eu -avirasecureserver.com -ayerbo.xhost.ro -babosas.com -bad-stars.net -baikaleminer.com -bambi.ck.ua -banki76.ru -banksrf.ru -batanga.net -battlecarnival.su -bestbrainprod.win -bestpornuha.com -bestwrinklecreamnow.com -bget.ru -bgviagrachrx.com -bimatoprost-careprost.com -bio-market.kz -bioinnovate.co -bird1.ru -bitporno.sx -bjgugu.net.ua +aviav.org +awaybird.ru +ayeartoforget.com +ayodhya.co +azazaporn.com +bankmib.ru +bannerspace.com +bavsac.com +best-way.men +bestofferswalkmydogouteveryday.gq +bestpriceninja.com +bestssaker.com +beztuberkuleza.ru +bidr.trellian.com +bitcoin-ua.top +bizru.info blackcurranthumidifiers.site -blockety.co +blobar.org +blog.f00kclan.de +bloggers.nl blue-square.biz -bmw.afora.ru -boom-celebs.com -boostmyppc.com -brandedleadgeneration.com -brandimensions.com -break-the-chains.com -breastaugmentation.co.za -buehne-fuer-menschenrechte.de +bluerobot.info +bluesalt.co +bmsco.co +boanonihaca.tk +bombla.org +bonvillan.com +bookmaker-bet.com +bradleylive.xyz +brandehk.dk +brokergid.ru buketeg.xyz -bupropion-sr-150-mg.us -calstaterealty.us +buron.pw +business-made-fun.com +buyessaynow.biz +buynorxx.com +buzz-porno.info +buzzurl.jp +call-of-duty.info +camdenmemorials.com +canoncdriverq3.pen.io +cardul.ru cartierlovestore.com -cerev.info -chatroulette.world +cartierreplica.win +caveavins.fr +cdncash.com +cercacamion.it +cheapjerseysfootballshop.com +checkhit.com +checktext.org chinaelitecheapjerseys.com -chlooe.com -ci.ua -clash-clans.ru -clickintext.net -coaching-netz.info -cognitiveseo.com -compliance-andrew.xyz -compliance-checker.info -compliance-elena.top -compliance-jane.top -connectingsingles.com -continent-e.tv +cineacademy.ru +cjs.com.ru +clarithromycin500mg.com +clippingphotoindia.com +clknsee.com +clodo.ru +com-cleaner.systems +communisave.co.za +compliance-brian.top +compliance-jess.top +compliance-olga.top +concealthyself.com +contextualyield.com cookingmeat.ru -credit-online.ws -culturevie.info -customboxes4less.com -data.vtc.pw -dating-app.ru -dating-time-now.com -daymusam.com -dekoration.us +corta.co +craftburg.ru +crafthubs.com +crazyprotocol.com +creativehutindia.com +crystalslot.com +cutt.us +cxpromote.com +d-black.bz +data-mining.tk +datsun-do.com +dcj-nn.ru +decorazilla.com delfin-aqua.com.ua -deplim.com -deshevo-nedorogo.ru -detalizaciya-tut.biz -digital-sale.su -dimfour.com -diminishedvalueoforegon.com -distver.ru -diusyawiga.tk -divku.ru -doctissimo.fr -doctormakarova.ru -dogbreedspicture.net -domainsphoto.com -dominterior.org -doreenblog.online -downloads-whatsapp.com -dptaughtme.com -drunkenstepfather.com -druzhininevgeniy63.blogspot.com -dumuelave.xyz -dvd-famille.com -dverimegapolis.ru -e-avon.ru -ecommerce-seo.org -ecookna.com.ua -ed-shop01.ru -eduserver.net -eets.net -ekobata.ru -electro-prom.com -elektrozigarettekaufen2.blogse.nl -elitepcgames.com -elmifarhangi.com +depositfiles-porn.ga +destinationrealestate.com +digilander.libero.it +docsportal.net +dojki-hd.com +domain-submit.info +downloadmefiranaratb1972.xpg.com.br +e705.net +easytuningshop.ru +edge.sharethis.com +ekspertmed.com +elektronischezigarette2.mex.tl +elektrozigarette2.webs.com +emergencyneeds.org +enbersoft.com encodable.com -englishgamer.com -enskedesquashclub.se -erotische-geschichten-xxl.com -escortslet.net -europeanwatches.ru -exchangeit.gq -exdocsfiles.com -ezofest.sk -family1st.ca -faracontrol.ir +eng-lyrics.com +eredijovon.com +esopini.com +estelight.ru +exmasters.com +exonline.info +extremez.net +ezigarettekaufen2.blox.pl +ezigarettenshop2.postbit.com +f-loaded.de +f-online.de +fable.in.ua +facecup.top +facialporntube.com +familyholiday.ml felizporno.com -fickenprivat.info -fikasound.tk -filesclub.net -filkhbr.com -fillmewithhappiness.com -filmgo.ru -firma-legion.ru -fordsonmajbor.cf -forex21.ru -fortunejack.com +fifa-coins.online +figuringmoneyout.com +filesdatabase.net +financetip.eu +fitness-video.net +flooringinstallation-edmonton.com +florida-tourism.net fortwosmartcar.pw -freefoto.ca -friendflnder.com -frustrated-favorable.gq -fxund.us -gallerylisting.com +forum.poker4life.ru +foto-sisek.porngalleries.top +fr-bearings.ru +free-share-buttons.com +free-share-buttons.top +freeuploader.com +fres-news.com +freshdz.com +frighteningremain.cf +fsalas.com +fuel-gas.com +galaxyflowers.ru +game300.ru +gaming-journal.com +gaz-voshod.ru ge0ip.net -gembird.com -gerhardhealer.com -gfaq.ru -gg.zzyjxs.com -gilsonchiro.xyz -glopages.ru -gloverid.site -gogps.me -goodprotein.ru -googglet.com +gentamicineyedrops.blogspot.com +get-free-traffic-now.com +gettpromos.com +ghernnqr.skyrock.com +gifspics.com +girls-ufa.ru +glicol.kz +globetrotting-culture.ru +go2album.com +goldenggames.com +goodnightjournal.com +goodwinmetals.co +gpms.org.my +gq-catalog.gq +greatfind-a.akamaihd.net +greatzip.com greendream.com.ua -grizzlysgrill.com -growmyfunds.ca -gsasearchengineranker.pw +gsasearchengineranker.site gta-top.ru -helicalpile.us -hermesbracelets.xyz -hobild.net -homemade.gq -hotchatdate.com +gwebtools.com +hanwei.us +hard-porn.mobi +hccoder.info +hdimagelib.com +heatpower.ru +hjaoopoa.top +hobbyhuren24.net hotloans.ru -houseofgaga.ru -houston-vikings.com -howtoclean.club -i-hobot.ru -ibb.com.ua -icaseclub.ru -ictizanidinehcl4mg.blogspot.com -ideashome.id -ideawheel.com -ilmexico.com -imagui.eu -immobiliaremassaro.com -imperia31.ru -import-sales.com -in-tandem.co -intermesh.net -iopeninghours.co.uk -irkutsk.online-podarki.com -jamiembrown.com -jav-fetish.site +hptwaakw.blog.fc2.com +hustoon.over-blog.com +i-service.kz +ideibiznesa2015.ru +ifirestarter.ru +images.gyffu.com +imgata.com +impotentik.com +indianmedicaltourismshop.com +indo-export.ru +infektsii.com +infostatsvc.com +infoupdate.org +insta-add.pro +internet-apteka.ru +investyb.com +iqoption-bin.com +iqoption.pro +isitpaleo.info +islamtoday.co.za +itch.io +itis4you.com +ivearchenceinflu.cf +iwantmyfreecash.com +jacago.com +jerseychinabizwholesale.com jerseychinabizwholesale.us -jerseyswholesalechinalimited.com -joessmogtestonly.com -journeydownthescale.info -juliaworld.net -jwss.cc +jmat.cn +jofucipiku.tk +jpcycles.com +jyrxd.com kadashihotel.com -kam-dom.ru -kaz.kz -kenaba.su -keywords-monitoring-success.com -keywordspay.com -kinobaks.com -kinobest.pl -kirov.zrus.org -kmgamex.cf -kolotiloff.ru +kambasoft.com +katushka.net +kino-rf.ru +klin3952.ru +konkursowo-24.pl kountrylife.com -krasnodar.zrus.org -kriokomora.info -kunstaktien.info -laapp.com -lacave.ntic.fr -languagecode.com -lapolis.it -law-check-eight.xyz -lawyers.cafe -lecbter-relationships.ga -letmacworkfaster.site -lietaer.com -life.biz.ua -likesdesign.com -linksharingt.com -lirunet.ru -lisque.batcave.net +kraljeva-sutjeska.com +krasivoe-hd.com +krasivoe-hd.net +ks1234.com +kurbappeal.info +kustanay.kz +kvartiry-remont.ucoz.ru +ladov.ru +landscapebackgrounds.blogspot.com +larchik.net +letslowbefast.site +lifehacĸer.com +lignofix.ua +lineavabit.it +listiseltemournlan.gq lob.com.ru -longgreen.info -love-baby.cz +lotto6888.com lulea-auktionsverk.se -magicdiet.gq -magnetpress.sk -maladot.com -mantramusic.ru -marktforschung-stuttgart.com -marsgatan.com -mathgym.com.au -medicineseasybuy.com -medicovi.com +macotool.com +maju.bluesalt.co +master-muznachas.ru +matb3aa.com +mcadamssupplyco.com medispainstitute.com.au -meetlocalchicks.com -metascephe.com -migente.com -mikozstop.com -mikrobiologies.ru -mindeyegames.com -minharevisao.com +megakino.net +metabar.ru +mielec.pl +millionare.com +mir-limuzinov.ru mirmedinfo.ru -mixtapetorrent.com -mlf.hordo.win -mobile.ok.ru -mockupui.com -modnie-futbolki.net -moetomnenie.com -moi-glazki.ru +mirtorrent.net +mnogolok.info +modabutik.ru moje-recenze.cz -mole.pluto.ro -monclerboots.xyz -monclerjacketsoutlet.win -monclervests.xyz -monitorwebsitespeed.com -mostantikor.ru -mototsikl.org -mountainstream.ms -movies-in-theaters.net -moyaterapiya.ru +mokrayakiska.com +montazhnic.ru +moomi-daeri.com +moroccosurfadventures.com +mrcsa.com.au +msk-diplomat.com muizre.ru -musirc.com -mustat.com -my-aladin.com -mycaf.it -myplaycity.com +mydearest.co +mystats.xyz nagdak.ru -najaden.no -nakrutka.cc -napalm51.nut.cc -naphukete.ru naturalpharm.com.ua -negociosdasha.com -nelc.edu.eg -new-post.tk -nicefloor.co.uk -nl.netlog.com -nootrino.com -npoet.ru -nvformula.ru -oakleyglassesonline.us -ofanda.com -online-sbank.ru -onlineku.com -optom-deshevo.ru -orgasmus-virtual.com -orion-v.com +naturtreenspicerx.pw +naval.jislaaik.com +net-profits.xyz +newstraveller.ru +nhl17coins.exblog.jp +niroo.info +nmrk.ru +nodup.ru +noumeda.com +nudejapan.net +o00.in +obsessionphrases.com +odesproperty.com +office2web.com +offtime.ru +onlywoman.org +osnova3.ru +ostroike.org ozoz.it -partnerworkroom.men -parvezmia.xyz -pepperstyle.ru -picphotos.net -pigrafix.at -piulatte.cz -pl-vouchers.com -pl.aasoldes.fr -playboyfiles.xblog.in -pliks.pl -poems.com.ua -polybuild.ru -popmarker.com -popunder.ru -pornogig.com -pornokajf.com -pornotubs.com -pornoxxx.com.mx -postclass.com -pozdravleniya-c.ru -pozvonim.com -priscilarodrigues.com.br -privetsochi.ru -profitwithalex.info -prom23.ru -proposal-engine.com -prostitutki-kharkova.org -pukaporn.com -purplestats.com -pwwysydh.com -pyramidlitho.webs.com -quick-seeker.com -razamicroelectronics.com -registrationdomainsite.com -remedyotc.com +papasdelivery.ru +pattersonsweb.com +pc4download.co +pcgroup.com.uy +pechikamini.ru +phrcialiled.com +pio.polytopesexempt.com +pixelrz.com +pkr1hand.com +pl.id-forex.com +pomoc-drogowa.cba.pl +popander.mobi +popugaychiki.com +poquoson.org +pornmania.pl +pornolook.net +pornpost.in +prchecker.info +predominant-invent.tk +prize44.com +profitkode.com +progress-upakovka.ru +prospekt-st.ru +prostitutki-tolyatti.org +pts163.ru +pxhdwsm.com +qld10000.net +qor360.com +questionmarque.ch +rapidsites.pro +raschtextil.com.ua +rating-bestcasino.com +rating-casino2021.ru +realtytimes.com +reklama-i-rabota.ru +reklamuss.ru +remontgruzovik.ru rennlist.com -rentalcarnavi.info -rentaremotecomputer.com -repeatlogo.co.uk +research.ifmo.ru reversing.cc -rootandroid.org -rosbalt.com.ua -royal-investments.net -rubanners.com -runtnc.net -s8-nowy-wygraj.comli.com -savememoney.co.za -sbetodiodnye-lampy.ru -scanner-don.top -scanner-elena.top -scanner-jane.top -scanner-mary.top -scottbywater.com -search-goo.com -search.alot.com -searchimage.co -see-your-website-here.com -semaltmedia.com -seminarygeorgia59.ga -semrush.com -seocheckupx.com -seojokes.net -seotoolsagency.com -sex-tracker.de -sexkontakteao.info -sharpchallenge.com -shohanb.com -sigmund-freud.co.uk -sildenafil-tadalafil.info -simul.co -siteaero.com -skincrate.net -slavkokacunko.de -smartbalanceworld.com -snip.to -softtor.com -solinf.co -soochi.co -spb.afora.ru -srecorder.com -stal-rulon.ru -statustroll.com -stpolice.com -stroydetali.ru -strv.se -students-cheapskate.ml -sudexpert66.ru -supermodni.com.ua -support.nopeas.sk -svarbit.com -svarkagid.com -svetlotorg.ru -tackletarts.co -taihouse.ru +rossanasaavedra.net +rrutw.com +rubbed.us +ruf777.com +ruspdd.com +russintv.fr +rustic-quiver.win +sadaholding.com +saneitconsulting.com +sankt-peterburg.nodup.ru +santechnik.jimdo.com +sashagreyblog.ga +scalerite.co.za +scanner-barak.top +scanner-julia.top +scanner-marwin.top +schlampen-treffen.com +scopich.com +search.xtconnect.com +seccioncontrabajo.com +seinterface.com +seksvideoonlain.com +semprofile.com +seoheap.com +sergiorossistore.online +serviporno.com +sex-pr.net +sexpartygirls.net +sextracker.be +sexystrippe.info +shiksabd.com +signx.info +similardeals.net +sitiz.club +sk.golden-praga.ru +sledstvie-veli.net +slomm.ru +smichovbike.cz +sms2x2.ru +soc-econom-problems.ru +social-buttons.xyz +soft-program.com +sonnikforme.ru +souvenir.cc +spammen.de +sport7777.net +stair.registrydomainservices.com +star61.de +stretchmate.net +strigkaomsk.ru +studentguide.ru +sukarame.net +superiends.org +supers.com.ua +surintech.ac.th +svensk-poesi.com +swagbucks.com +t-bygg.com +takethatad.com +tandvardshuset.net td-l-market.ru -tecnoteakviareggio.it -teenfuck.tv -telemetryverification.net +tds-advert005.info +teenporn18.net +teesdaleflyballclub.co.uk +terraclicks.com +terrafootwear.us teslathemes.com -theendivechronicles.com -thegameriders.com -themestotal.com -thexart.club -threecolumnblogger.com -titelhelden.eu -tizanidine4mgstreetvalue.blogspot.com -tizanidineduringpregnancy.blogspot.com -tizanidinestreetvalue.blogspot.com -tizanidineusedfor.blogspot.com -tmtrck.com -token-lab.org -toon-families.com -toproadrunner5.info -torrentgamer.net -toxicwap.com -tracksz.co -trafficmania.com -trahic.ru -treasuretrack-a.akamaihd.net -trion.od.ua -trubywriting.com -trustedmaccleaner.com -tube8.com -turizm.bz -turizmus.us -tvoystartup.ru -twiclub.in -twittrading.com +tetracsaudi.com +thebitcoincode.com +thebluenoodle.com +thefds.net +theguardlan.com +thenews-today.info +thepantonpractice.co.uk +theprofitsmaker.net +titangel-vietnam.com +tizanidinehcl4mgisitanarcotic.blogspot.com +tizanidinestreetprice.blogspot.com +tizanidinevsvalium.blogspot.com +tomatis.gospartner.com +toondinsey.com +top1-seo-service.com +topdownloads.ru +torrentred.games +torrents.cd +totu.info +touchmods.fr +toursmaps.com +track112.site +traxdom.ru +tuckpointingmasonrysystems.com +u-cheats.ru +udsgame.online +ufa.xrus.org umg-stroy.ru -upproar.com -uptimechecker.com -urdoot.win -url-extractor.xyz -usswrite.com -ustion.ru -v720hd.ru -vashsvet.com -viagengrarx.com -viandpet.com +unece.org +uprour.com +uptime.com +url-img.link +usbggettwku.ga +valsalud.com +valuado.com +varikozdok.ru +venta-prom.ru +vepad.com +vesnatehno.com +vgoloveboli.net +viagra-soft.ru videochat.bz -videochaty.ru -vigrx-original.ru -viktoria-center.ru -vip2ch.com -vipps.com.my -vladimir.xrus.org -voloomoney.com -vvpg.ru -vzglyadriv.kg +videogamesecrets.com +vipsexfinders.com +vita.com.hr +vitalads.net +vk-mus.ru +vsdshnik.com +vse-pesni.com +vvon.co.uk w-journal.ru -wallpaperaccess.com -walpaperlist.com -warezkeeper.com -watchmygf.net -wdfdocando.com +wardreapptokone.tk +warezaccess.com wdrake.com -webscouter.net -wellcome2slovenia.ru -wesharepics.site -whatsappdownload10.com -while.cheapwebsitehoster.com -whiteelephantwellington.com -whosonmyserver.com +wechatdownload10.com +wheelchairliftsarea.com +winx-play.ru +wleuaprpxuvr.ga wma-x.com -wnhjavlhezp.gq -word-vorlagen.xyz -www.ehscloud.cn -www.kabbalah-red-bracelets.com -x-lime.com -x-lime.net -xboxster.ru -xn-----6kcaabbafhu7cskl7akvongwpo7hvjj.xn--p1ai -xn--80ak6aa92e.com -xn--b1addnj3cah.xn--p1ai -xnxx-n.com +workle.website +worldtraveler.world +wrona.it +ws.ampower.me +wygraj-skiny.win +xchat26.myfreecams.com +xerox-douglas.cf +xlovecam.com +xn------7cdbapdecfd4ak1bn0amjffj7afu3y.xn--p1ai +xn----7sbahjd3btneuw1joc.xn--p1ai +xn----7sbbagbq7bd5aheftfllo4m.xn--p1ai +xn----7sbho2agebbhlivy.xn--p1ai +xn--24-glceagatoq7c2a6ioc.xn--p1ai +xn--90acenikpebbdd4f6d.xn--p1ai +xxxkaz.org xxxsiterips.xyz -youjizz.vc -youporn-ru.com -youradexchange.com -yourads.website -yourdesires.ru +yaaknaa.info +yellocloud.be +yes-com.com yourtemplatefinder.com -youthreaders.com -zaimhelp.ru -zaloro.com -zeroredirect.com -zhongwenlink.com -zirondelli.it +yurgorod.ru +z-master.ru +zajm-pod-zalog-nedvizhimosti.ru +zakon-ob-obrazovanii.ru +zaloadi.ru +zhacker.net +znakomstva-moskva77.ru +zscalerone.net +zscalertwo.net diff --git a/dev-tools/test_units/random-referrers-for-whitelist-test.list b/dev-tools/test_units/random-referrers-for-whitelist-test.list index 236d18195..efeb78686 100644 --- a/dev-tools/test_units/random-referrers-for-whitelist-test.list +++ b/dev-tools/test_units/random-referrers-for-whitelist-test.list @@ -1,250 +1,250 @@ -0c47f8422d3f.com -0daymusic.org -3weekdiet.com -4senses.co -5kstore.com -aasoldes.fr -acgs.tk +1-99seo.com +12masterov.com +91abcw.com +aa9d046aab36af4ff182f097f840430d51.com +aaa38852e886ac4af1a3cff9b47cab6272.com actices.com -actulite.com -adelly.bg -adobereader-free.ru -ahhjf.com -akuhni.by -alldownload.pw -anal-acrobats.hol.es -anjalika.co.in -api.stathat.com -apibetweenlinesn-a.akamaihd.net -appmsr.org -aproposde.com -aquarium-pro.ru -aramaicmedia.org -ascat.porn -auto-zapchasti.org -autobrennero.it -avon-novosib.ru -bablonow.ru -bankiem.pl -belinvestdom.by -bestcoin.cc -biketank.ga -bio-market.kz -bloggers.nl -bookmark4you.com.biz -bosman.pluto.ro -breastaugmentation.co.za -burger-tycoon.com -bus-offer.com -butstrap.space -cartechnic.ru -cartierloveringreplica.xyz -cartierreplica.win -cheapmoncler.win -cialischmrx.com -cinemaflix.website -cjs.com.ru -cleanmypc.com -clickpapa.com -clippingphotoindia.com -cntravelre.com -coldfilm.ru -collegerentals.com -compliance-ivan.top -coolbar.pro -coolgamechannel.com -coop-gamers.ru -cornomase.win -cracksplanet.com -creativehutindia.com -crirussian.ru -cubs-jerseys.us -custom-product-labels.com -daptravel.com -ddpills.com -dealighted.com -dekoration.us -djonwatch.ru -dogoimage.com -dokumentalkino.net -downloadkakaotalk.com -draniki.org -earthmagic.info -edge.sharethis.com -elektir.ru -elektrischezigarette1.onsugar.com -elmifarhangi.com -energydiet24.ru -erotik-kostenlos.net -essay-writing.work -eywords-monitoring-your-success.com -f07.de -faptitans.com -faracontrol.ir +adsloads.com +afftrack001.com +afslankpillen2017nl.eu +albumsuper.info +alert.scansafe.net +alloysteel.ru +amazon-adsystem.com +ameblo.top +android4fun.org +apishops.ru +appiq.mobi +architecturebest.com +aruplighting.com +ausergrubhof.info +automobile-spec.com +avtoarenda.by +avtochehli.by +baltgem.com +beget.tech +bigcities.org +bimatoprost-careprost.com +bin-brokers.com +bitcoinpile.com +blacktwhite.com +bmw.afora.ru +brothers-smaller.ru +bryansk.zrus.org +businescoop.men +buynorxx.com +bytimedance.ru +bzero1jewelry.net +cactussoft.biz +camel-beach.com +canrioloadm.gq +checkm8.com +cherubinimobili.it +chocolateslim-en-espana.com +chocolateslim-en-france.com +chorus.terakeet.com +cicaki.net +clicksor.com +coaching-netz.info +czeshop.info +daymusam.com +defenderxtactical.com +demian.kz +descargar-musica-gratis.net +diegolopezcastan.com +dieta-personalna.pl +dimaka.info +djihispano.com +dnsrsearch.com +docs4all.com +drev.biz +efkt.jp +electro-prom.com +elektrozigaretten1.webs.com +encodable.com +enginebay.ru +event-tracking.com +extener.org +f5mtrack.com +fast-torrent.ru fickenprivat.info -filesmonster.porn -fitness-video.net -flauntyoursite.com -forensicpsychiatry.ru -forsex.info -forum-engineering.ru -foto-sisek.porngalleries.top -front.ru -fun-mobi.pl -gagrasector.ru -game300.ru -gamevalue7.weebly.com -getresponse.com -glcomputers.ru -golden-catalog.pro -good-mummy.ru -gotorussia.com -gq-catalog.gq -graphicwe.org -gratis-sexkontakte.com -greatfind-a.akamaihd.net -guigyverpo.cf -hahashka.ru +firstsiteguide.com +fr.netlog.com +freejabs.com +gamesprite.me +gearsadspromo.club +get-free-social-traffic.com +ghostvisitor.com +gidonline.one +gigixo.com +glogow.pl +gribkovye-zabolevaniya.com +gribokstop.com +gsasearchengineranker.space +gta-top.ru hamilton.ca -hannasolution.ru -haveinc.xyz -hdwallpapers-free.com -hotelcrocenzi.sm -hotsocialz.com -idc.com.ua -idealtits.net -immobiliaremassaro.com -import-sales.com -inbabes.sexushost.com -inboundlinks.win -infogame.name -isitpaleo.info -jpcycles.com +hasshe.com +hitmuzik.ru +hmywwogw.bloger.index.hr +honyaku.yahoofs.jp +hotenergy.ru +htmlcorner.com +ictizanidinehcl4mg.blogspot.com +ikritikimou.gr +ilovevitaly.xyz +images.gyffu.com +inet-traffic.com +intimshop-fantasy.ru +inzn.ru +javatex.co.id +jennyfire.ru +jmat.cn justbcause.com -jwss.cc -krasivye-devushki.net -ktotut.net -kunstaktien.info -kwzf.net -lampokrat.ws -leadn.pl -life.biz.ua -lineshops.biz -lion.snapmap.us -mabdoola.blogspot.com -magda-gadalka.ru -magiadinamibia.blogspot.com -mahnwachen-helfen.info -maridan.com.ua -mazda-roadsters.com -med-zdorovie.com.ua -mediaoffers.click -michaelkorsoutlet.store -michaelkorsoutletstore.net -minecraft-rus.org -mini.7zap.com -mixtapetorrent.com -monclervests.xyz -mrmoneymustache.com -mtmtv.info -mymercy.info -najaden.no +karadene.com +kenaba.su +keywordsuggest.org +kinohit1.ru +kirov.zrus.org +knowsitall.info +kongruan.com +kountrylife.com +krasnodar.xrus.org +landscaping.center +lasvegaslockandsafe.com +ldrtrack.com +letsart.ru +listiseltemournlan.gq +macdamaged.tech +magicalfind-a.akamaihd.net +mainhp.net +manifestation.betteroffers.review +mapquestz.us +markjaybeefractal.com +marsgatan.com +matematikus.info +matrixalchemy.com +max-p.men +maxthon.com +milblueprint.com +mirtorrent.net +mobile-appster.ru +mobplayer.net +moetomnenie.com +montazhnic.ru +morf.snn.gr +mosrif.ru +mossmesi.com +mycouponizemac.com +mydownlodablefiles.com +naturalshair.site netfacet.net -nikhilbahl.com -norththeface.store -nottyu.xyz -obuv-kupit.ru -okeinfo.online -papasdelivery.ru -partnerads.men -pb-dv.ru -pcads.ru -pestomou.info -pflexads.com -php-market.ru -physfunc.ru -piuminiita.com -pliks.pl -popup-fdm.xyz -pornmania.pl -pornoelita.info -pretty-mart.com -privat-girl.net -rankscanner.com -redhotfreebies.co.uk -registrydomainservices.com -remote-dba.de -renhacklids.tk -ringtonepartner.com -ruspdd.com -sajatvelemeny.com -sale-japan.com -sandhillsonline.com -santasgift.ml -santechnik.jimdo.com +netpics.org +networkcheck.xyz +nibbler.silktide.com +nsatc.net +nvformula.ru +offtime.ru +old-rock.com +online-x.ru +onlinefilmz.net +ooo-gotovie.ru +ostrovtaxi.ru +oynat.info +paighambot.com +pandroid.co +photokitchendesign.com +pinwallpaper.xyz +piski.top +pixshark.com +popads.net +porno-play.net +pornogig.com +pornolook.net +privacylocationforloc.com +privatov-zapisi.ru +productarium.com +proftests.net +progress-upakovka.ru +pronorm.fr +rabot.host.sk +ranksonic.com +recipedays.com +reeyanaturopathy.com +refudiatethissarah.info +reklama-i-rabota.ru +remont-comp-pomosh.ru +responsive-test.net +safe-app.net +saletool.ru +samara.rosfirm.ru sarahmilne.top -sbf441.com -scanner-andrew.top -scanner-barak.top +scanner-irvin.top scanner-marwin.top -seansonline24.pl -search.alot.com -searchimpression.com -seogadget.ru -seorank.info -seorankinglinks.xyz -service-core.ru -sexblog.pw -sexpartygirls.net -sexuria.net -sexy.babes.frontend-stack.top -shijian.ac.cn -sitopreferito.it -smallseotools.com -social-s-hhh.xyz -softomix.net -spammen.de -ssconstruction.co -sstroy44.ru +seccioncontrabajo.com +seo-traffic-ranking.info +seorussian.ru +servethis.com +serviporno.com +sexsearch.com +sexvporno.ru +shemalegalls.blogporn.in +sherlock.se +sibvitr.ru +sklad-24.ru +slim.sellany.ru +slutloadlive.com +smartbalanceworld.com +socialbookmarksubmission.org +softxaker.ru +soheavyblog.com +solmarket.by +sonata-arctica.wz.cz +soserfis.com +sotechco.co +sptslmtrafms.com +spywareit.com +sr-rekneskap.no +statoutlook.info +stauga.altervista.org +sterva.cc stevemonsen.com -stop-gepatit.te.ua -surfbuyermac.com -svolze.com -thin.me.pn -tizanidine4mgstreetvalue.blogspot.com -tizanidineusedfor.blogspot.com -tizanidinevssoma.blogspot.com -toonfamilies.net -topads.men -trafficjunky.net -trafficmonetize.org -tubeoffline.com -tvory.predmety.in.ua -uchetunet.su -ufa.xrus.org -ugguk.online -umekana.ru -unionmarkt.de -uogonline.com -vapeface.club -varikozdok.ru -videooko.weebly.com -vucms.com -website-datenbank.de -website-speed-checker.site -welcomeauto.ru -weprik.ru -wesharepics.info -woodyguthrie.se -www.timer4web.com -xbaboon.com -xlovecam.com -xn--80aanaardaperhcem4a6i.com -zahvat.ru -zarepta.com -zeroredirect11.com -zeroredirect9.com -zigzog.ru -zlatnajesen.com -znaturaloriginal.com -zvezdagedon.ru +stuff-about-money.com +swagbucks.com +taqywu51.soup.io +tecnoteakviareggio.it +telefonsexkostenlos.tk +template-kid.com +thefotosgratis.eu +thewebsitetemplate.info +top-karkas.ru +top-l2.com +treasuretrack-a.akamaihd.net +tsstcorpcddvdwshbbdriverfb.aircus.com +udayavani.com +underthesite.com +urlopener.blogspot.com.au +wahicbefa31.soup.io +web-analytics.date +weblo.com +whatsappbot.flyland.ru +wildworld.site +wordpress-crew.net +wufak.com +x-diesel.org +xcc24.pl +xjrul.com +xn-----6kccaibs5cb8afhjrfmix2n.xn--p1ai +xn----dtbndd4ae7eub.top +xn--1-8sbcpb0bdm8k6a.xn--p1ai +xn--80aaaks3bbhabgbigamdr2h.xn--p1ai +xn--80aafb2a.xn--p1ai +xn--d1acah0c.xn--p1ai +xn--lifehacer-1rb.com +xpresscare.ru +yginekologa.com +you-shall-not-pass.is74.ru +youbrainboost.asia +zajm-pod-zalog-nedvizhimosti.ru +zawyna.ua +zeroredirect10.com +zhacker.net diff --git a/dev-tools/test_units/ratelimittest-master.txt b/dev-tools/test_units/ratelimittest-master.txt index b2122ba99..3656720f2 100644 --- a/dev-tools/test_units/ratelimittest-master.txt +++ b/dev-tools/test_units/ratelimittest-master.txt @@ -1,11 +1,21 @@ - + + + +Welcome to NginX + + +
+

Welcome to the NginX Ultimate Bad Bot Blocker

+

This is merely used for testing !!!

+ + 503 Service Temporarily Unavailable

503 Service Temporarily Unavailable

nginx/1.18.0 (Ubuntu)
-ml> + 503 Service Temporarily Unavailable

503 Service Temporarily Unavailable

diff --git a/dev-tools/test_units/whitelist-ips.conf b/dev-tools/test_units/whitelist-ips.conf index d47ae2e04..170f5965e 100644 --- a/dev-tools/test_units/whitelist-ips.conf +++ b/dev-tools/test_units/whitelist-ips.conf @@ -1,2 +1,2 @@ -10.1.0.198 0; +10.1.0.132 0; 127.0.0.1 0; diff --git a/dev-tools/tmp/robots.txt b/dev-tools/tmp/robots.txt index 1234aff9f..5342c2879 100644 --- a/dev-tools/tmp/robots.txt +++ b/dev-tools/tmp/robots.txt @@ -53658,3 +53658,1255 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2494 +### Updated: Sat Jul 31 14:05:41 UTC 2021 +### Bad Bot Count: 617 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ALittle Client +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: Leap +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: ubermetrics-technologies.com +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ + diff --git a/robots.txt/robots.txt b/robots.txt/robots.txt index 1234aff9f..5342c2879 100644 --- a/robots.txt/robots.txt +++ b/robots.txt/robots.txt @@ -53658,3 +53658,1255 @@ Disallow:/ User-agent: zgrab Disallow:/ +################################################################### +# The Ultimate robots.txt Bot and User-Agent Blocker +# Copyright: +# https://github.com/mitchellkrogza/nginx-ultimate-bad-bot-blocker +################################################################### + +### Version Information # +################################################### +### Version: V4.2021.07.2494 +### Updated: Sat Jul 31 14:05:41 UTC 2021 +### Bad Bot Count: 617 +################################################### +### Version Information ## + +User-agent: * +Disallow: /wp-admin/ +Allow: /wp-admin/admin-ajax.php +User-agent: 01h4x.com +Disallow:/ +User-agent: 360Spider +Disallow:/ +User-agent: 404checker +Disallow:/ +User-agent: 404enemy +Disallow:/ +User-agent: 80legs +Disallow:/ +User-agent: ADmantX +Disallow:/ +User-agent: AIBOT +Disallow:/ +User-agent: ALittle Client +Disallow:/ +User-agent: ASPSeek +Disallow:/ +User-agent: Abonti +Disallow:/ +User-agent: Aboundex +Disallow:/ +User-agent: Aboundexbot +Disallow:/ +User-agent: Acunetix +Disallow:/ +User-agent: AfD-Verbotsverfahren +Disallow:/ +User-agent: AhrefsBot +Disallow:/ +User-agent: AiHitBot +Disallow:/ +User-agent: Aipbot +Disallow:/ +User-agent: Alexibot +Disallow:/ +User-agent: AllSubmitter +Disallow:/ +User-agent: Alligator +Disallow:/ +User-agent: AlphaBot +Disallow:/ +User-agent: Anarchie +Disallow:/ +User-agent: Anarchy +Disallow:/ +User-agent: Anarchy99 +Disallow:/ +User-agent: Ankit +Disallow:/ +User-agent: Anthill +Disallow:/ +User-agent: Apexoo +Disallow:/ +User-agent: Aspiegel +Disallow:/ +User-agent: Asterias +Disallow:/ +User-agent: Attach +Disallow:/ +User-agent: AwarioRssBot +Disallow:/ +User-agent: AwarioSmartBot +Disallow:/ +User-agent: BBBike +Disallow:/ +User-agent: BDCbot +Disallow:/ +User-agent: BDFetch +Disallow:/ +User-agent: BLEXBot +Disallow:/ +User-agent: BackDoorBot +Disallow:/ +User-agent: BackStreet +Disallow:/ +User-agent: BackWeb +Disallow:/ +User-agent: Backlink-Ceck +Disallow:/ +User-agent: BacklinkCrawler +Disallow:/ +User-agent: Badass +Disallow:/ +User-agent: Bandit +Disallow:/ +User-agent: Barkrowler +Disallow:/ +User-agent: BatchFTP +Disallow:/ +User-agent: Battleztar Bazinga +Disallow:/ +User-agent: BetaBot +Disallow:/ +User-agent: Bigfoot +Disallow:/ +User-agent: Bitacle +Disallow:/ +User-agent: BlackWidow +Disallow:/ +User-agent: Black Hole +Disallow:/ +User-agent: Blackboard +Disallow:/ +User-agent: Blow +Disallow:/ +User-agent: BlowFish +Disallow:/ +User-agent: Boardreader +Disallow:/ +User-agent: Bolt +Disallow:/ +User-agent: BotALot +Disallow:/ +User-agent: Brandprotect +Disallow:/ +User-agent: Brandwatch +Disallow:/ +User-agent: Buck +Disallow:/ +User-agent: Buddy +Disallow:/ +User-agent: BuiltBotTough +Disallow:/ +User-agent: BuiltWith +Disallow:/ +User-agent: Bullseye +Disallow:/ +User-agent: BunnySlippers +Disallow:/ +User-agent: BuzzSumo +Disallow:/ +User-agent: CATExplorador +Disallow:/ +User-agent: CCBot +Disallow:/ +User-agent: CODE87 +Disallow:/ +User-agent: CSHttp +Disallow:/ +User-agent: Calculon +Disallow:/ +User-agent: CazoodleBot +Disallow:/ +User-agent: Cegbfeieh +Disallow:/ +User-agent: CensysInspect +Disallow:/ +User-agent: CheTeam +Disallow:/ +User-agent: CheeseBot +Disallow:/ +User-agent: CherryPicker +Disallow:/ +User-agent: ChinaClaw +Disallow:/ +User-agent: Chlooe +Disallow:/ +User-agent: Claritybot +Disallow:/ +User-agent: Cliqzbot +Disallow:/ +User-agent: Cloud mapping +Disallow:/ +User-agent: Cocolyzebot +Disallow:/ +User-agent: Cogentbot +Disallow:/ +User-agent: Collector +Disallow:/ +User-agent: Copier +Disallow:/ +User-agent: CopyRightCheck +Disallow:/ +User-agent: Copyscape +Disallow:/ +User-agent: Cosmos +Disallow:/ +User-agent: Craftbot +Disallow:/ +User-agent: CrazyWebCrawler +Disallow:/ +User-agent: Crescent +Disallow:/ +User-agent: CrunchBot +Disallow:/ +User-agent: Curious +Disallow:/ +User-agent: Custo +Disallow:/ +User-agent: CyotekWebCopy +Disallow:/ +User-agent: DBLBot +Disallow:/ +User-agent: DIIbot +Disallow:/ +User-agent: DSearch +Disallow:/ +User-agent: DTS Agent +Disallow:/ +User-agent: DataCha0s +Disallow:/ +User-agent: DatabaseDriverMysqli +Disallow:/ +User-agent: Demon +Disallow:/ +User-agent: Deusu +Disallow:/ +User-agent: Devil +Disallow:/ +User-agent: Digincore +Disallow:/ +User-agent: DigitalPebble +Disallow:/ +User-agent: Dirbuster +Disallow:/ +User-agent: Disco +Disallow:/ +User-agent: Discobot +Disallow:/ +User-agent: Discoverybot +Disallow:/ +User-agent: Dispatch +Disallow:/ +User-agent: DittoSpyder +Disallow:/ +User-agent: DnyzBot +Disallow:/ +User-agent: DomCopBot +Disallow:/ +User-agent: DomainAppender +Disallow:/ +User-agent: DomainCrawler +Disallow:/ +User-agent: DomainSigmaCrawler +Disallow:/ +User-agent: DomainStatsBot +Disallow:/ +User-agent: Domains Project +Disallow:/ +User-agent: Dotbot +Disallow:/ +User-agent: Download Wonder +Disallow:/ +User-agent: Dragonfly +Disallow:/ +User-agent: Drip +Disallow:/ +User-agent: ECCP/1.0 +Disallow:/ +User-agent: EMail Siphon +Disallow:/ +User-agent: EMail Wolf +Disallow:/ +User-agent: EasyDL +Disallow:/ +User-agent: Ebingbong +Disallow:/ +User-agent: Ecxi +Disallow:/ +User-agent: EirGrabber +Disallow:/ +User-agent: EroCrawler +Disallow:/ +User-agent: Evil +Disallow:/ +User-agent: Exabot +Disallow:/ +User-agent: Express WebPictures +Disallow:/ +User-agent: ExtLinksBot +Disallow:/ +User-agent: Extractor +Disallow:/ +User-agent: ExtractorPro +Disallow:/ +User-agent: Extreme Picture Finder +Disallow:/ +User-agent: EyeNetIE +Disallow:/ +User-agent: Ezooms +Disallow:/ +User-agent: FDM +Disallow:/ +User-agent: FHscan +Disallow:/ +User-agent: FemtosearchBot +Disallow:/ +User-agent: Fimap +Disallow:/ +User-agent: Firefox/7.0 +Disallow:/ +User-agent: FlashGet +Disallow:/ +User-agent: Flunky +Disallow:/ +User-agent: Foobot +Disallow:/ +User-agent: Freeuploader +Disallow:/ +User-agent: FrontPage +Disallow:/ +User-agent: Fuzz +Disallow:/ +User-agent: FyberSpider +Disallow:/ +User-agent: Fyrebot +Disallow:/ +User-agent: G-i-g-a-b-o-t +Disallow:/ +User-agent: GT::WWW +Disallow:/ +User-agent: GalaxyBot +Disallow:/ +User-agent: Genieo +Disallow:/ +User-agent: GermCrawler +Disallow:/ +User-agent: GetRight +Disallow:/ +User-agent: GetWeb +Disallow:/ +User-agent: Getintent +Disallow:/ +User-agent: Gigablast +Disallow:/ +User-agent: Gigabot +Disallow:/ +User-agent: Go!Zilla +Disallow:/ +User-agent: Go-Ahead-Got-It +Disallow:/ +User-agent: GoZilla +Disallow:/ +User-agent: Gotit +Disallow:/ +User-agent: GrabNet +Disallow:/ +User-agent: Grabber +Disallow:/ +User-agent: Grafula +Disallow:/ +User-agent: GrapeFX +Disallow:/ +User-agent: GrapeshotCrawler +Disallow:/ +User-agent: GridBot +Disallow:/ +User-agent: HEADMasterSEO +Disallow:/ +User-agent: HMView +Disallow:/ +User-agent: HTMLparser +Disallow:/ +User-agent: HTTP::Lite +Disallow:/ +User-agent: HTTrack +Disallow:/ +User-agent: Haansoft +Disallow:/ +User-agent: HaosouSpider +Disallow:/ +User-agent: Harvest +Disallow:/ +User-agent: Havij +Disallow:/ +User-agent: Heritrix +Disallow:/ +User-agent: Hloader +Disallow:/ +User-agent: Humanlinks +Disallow:/ +User-agent: HybridBot +Disallow:/ +User-agent: IDBTE4M +Disallow:/ +User-agent: IDBot +Disallow:/ +User-agent: IRLbot +Disallow:/ +User-agent: Iblog +Disallow:/ +User-agent: Id-search +Disallow:/ +User-agent: IlseBot +Disallow:/ +User-agent: Image Fetch +Disallow:/ +User-agent: Image Sucker +Disallow:/ +User-agent: IndeedBot +Disallow:/ +User-agent: Indy Library +Disallow:/ +User-agent: InfoNaviRobot +Disallow:/ +User-agent: InfoTekies +Disallow:/ +User-agent: Intelliseek +Disallow:/ +User-agent: InterGET +Disallow:/ +User-agent: InternetSeer +Disallow:/ +User-agent: Internet Ninja +Disallow:/ +User-agent: Iria +Disallow:/ +User-agent: Iskanie +Disallow:/ +User-agent: IstellaBot +Disallow:/ +User-agent: JOC Web Spider +Disallow:/ +User-agent: JamesBOT +Disallow:/ +User-agent: Jbrofuzz +Disallow:/ +User-agent: JennyBot +Disallow:/ +User-agent: JetCar +Disallow:/ +User-agent: Jetty +Disallow:/ +User-agent: JikeSpider +Disallow:/ +User-agent: Joomla +Disallow:/ +User-agent: Jorgee +Disallow:/ +User-agent: JustView +Disallow:/ +User-agent: Jyxobot +Disallow:/ +User-agent: Kenjin Spider +Disallow:/ +User-agent: Keyword Density +Disallow:/ +User-agent: Kinza +Disallow:/ +User-agent: Kozmosbot +Disallow:/ +User-agent: LNSpiderguy +Disallow:/ +User-agent: LWP::Simple +Disallow:/ +User-agent: Lanshanbot +Disallow:/ +User-agent: Larbin +Disallow:/ +User-agent: Leap +Disallow:/ +User-agent: LeechFTP +Disallow:/ +User-agent: LeechGet +Disallow:/ +User-agent: LexiBot +Disallow:/ +User-agent: Lftp +Disallow:/ +User-agent: LibWeb +Disallow:/ +User-agent: Libwhisker +Disallow:/ +User-agent: LieBaoFast +Disallow:/ +User-agent: Lightspeedsystems +Disallow:/ +User-agent: Likse +Disallow:/ +User-agent: LinkScan +Disallow:/ +User-agent: LinkWalker +Disallow:/ +User-agent: Linkbot +Disallow:/ +User-agent: Linkdexbot +Disallow:/ +User-agent: LinkextractorPro +Disallow:/ +User-agent: LinkpadBot +Disallow:/ +User-agent: LinksManager +Disallow:/ +User-agent: LinqiaMetadataDownloaderBot +Disallow:/ +User-agent: LinqiaRSSBot +Disallow:/ +User-agent: LinqiaScrapeBot +Disallow:/ +User-agent: Lipperhey +Disallow:/ +User-agent: Lipperhey Spider +Disallow:/ +User-agent: Litemage_walker +Disallow:/ +User-agent: Lmspider +Disallow:/ +User-agent: Ltx71 +Disallow:/ +User-agent: MFC_Tear_Sample +Disallow:/ +User-agent: MIDown tool +Disallow:/ +User-agent: MIIxpc +Disallow:/ +User-agent: MJ12bot +Disallow:/ +User-agent: MQQBrowser +Disallow:/ +User-agent: MSFrontPage +Disallow:/ +User-agent: MSIECrawler +Disallow:/ +User-agent: MTRobot +Disallow:/ +User-agent: Mag-Net +Disallow:/ +User-agent: Magnet +Disallow:/ +User-agent: Mail.RU_Bot +Disallow:/ +User-agent: Majestic-SEO +Disallow:/ +User-agent: Majestic12 +Disallow:/ +User-agent: Majestic SEO +Disallow:/ +User-agent: MarkMonitor +Disallow:/ +User-agent: MarkWatch +Disallow:/ +User-agent: Mass Downloader +Disallow:/ +User-agent: Masscan +Disallow:/ +User-agent: Mata Hari +Disallow:/ +User-agent: MauiBot +Disallow:/ +User-agent: Mb2345Browser +Disallow:/ +User-agent: MeanPath Bot +Disallow:/ +User-agent: Meanpathbot +Disallow:/ +User-agent: Mediatoolkitbot +Disallow:/ +User-agent: MegaIndex.ru +Disallow:/ +User-agent: Metauri +Disallow:/ +User-agent: MicroMessenger +Disallow:/ +User-agent: Microsoft Data Access +Disallow:/ +User-agent: Microsoft URL Control +Disallow:/ +User-agent: Mister PiX +Disallow:/ +User-agent: Moblie Safari +Disallow:/ +User-agent: Mojeek +Disallow:/ +User-agent: Mojolicious +Disallow:/ +User-agent: Morfeus Fucking Scanner +Disallow:/ +User-agent: Mozlila +Disallow:/ +User-agent: Mr.4x3 +Disallow:/ +User-agent: Msrabot +Disallow:/ +User-agent: Musobot +Disallow:/ +User-agent: NICErsPRO +Disallow:/ +User-agent: NPbot +Disallow:/ +User-agent: Name Intelligence +Disallow:/ +User-agent: Nameprotect +Disallow:/ +User-agent: Navroad +Disallow:/ +User-agent: NearSite +Disallow:/ +User-agent: Needle +Disallow:/ +User-agent: Nessus +Disallow:/ +User-agent: NetAnts +Disallow:/ +User-agent: NetLyzer +Disallow:/ +User-agent: NetMechanic +Disallow:/ +User-agent: NetSpider +Disallow:/ +User-agent: NetZIP +Disallow:/ +User-agent: Net Vampire +Disallow:/ +User-agent: Netcraft +Disallow:/ +User-agent: Nettrack +Disallow:/ +User-agent: Netvibes +Disallow:/ +User-agent: NextGenSearchBot +Disallow:/ +User-agent: Nibbler +Disallow:/ +User-agent: Niki-bot +Disallow:/ +User-agent: Nikto +Disallow:/ +User-agent: NimbleCrawler +Disallow:/ +User-agent: Nimbostratus +Disallow:/ +User-agent: Ninja +Disallow:/ +User-agent: Nmap +Disallow:/ +User-agent: Nuclei +Disallow:/ +User-agent: Nutch +Disallow:/ +User-agent: Octopus +Disallow:/ +User-agent: Offline Explorer +Disallow:/ +User-agent: Offline Navigator +Disallow:/ +User-agent: OnCrawl +Disallow:/ +User-agent: OpenLinkProfiler +Disallow:/ +User-agent: OpenVAS +Disallow:/ +User-agent: Openfind +Disallow:/ +User-agent: Openvas +Disallow:/ +User-agent: OrangeBot +Disallow:/ +User-agent: OrangeSpider +Disallow:/ +User-agent: OutclicksBot +Disallow:/ +User-agent: OutfoxBot +Disallow:/ +User-agent: PECL::HTTP +Disallow:/ +User-agent: PHPCrawl +Disallow:/ +User-agent: POE-Component-Client-HTTP +Disallow:/ +User-agent: PageAnalyzer +Disallow:/ +User-agent: PageGrabber +Disallow:/ +User-agent: PageScorer +Disallow:/ +User-agent: PageThing.com +Disallow:/ +User-agent: Page Analyzer +Disallow:/ +User-agent: Pandalytics +Disallow:/ +User-agent: Panscient +Disallow:/ +User-agent: Papa Foto +Disallow:/ +User-agent: Pavuk +Disallow:/ +User-agent: PeoplePal +Disallow:/ +User-agent: Petalbot +Disallow:/ +User-agent: Pi-Monster +Disallow:/ +User-agent: Picscout +Disallow:/ +User-agent: Picsearch +Disallow:/ +User-agent: PictureFinder +Disallow:/ +User-agent: Piepmatz +Disallow:/ +User-agent: Pimonster +Disallow:/ +User-agent: Pixray +Disallow:/ +User-agent: PleaseCrawl +Disallow:/ +User-agent: Pockey +Disallow:/ +User-agent: ProPowerBot +Disallow:/ +User-agent: ProWebWalker +Disallow:/ +User-agent: Probethenet +Disallow:/ +User-agent: Psbot +Disallow:/ +User-agent: Pu_iN +Disallow:/ +User-agent: Pump +Disallow:/ +User-agent: PxBroker +Disallow:/ +User-agent: PyCurl +Disallow:/ +User-agent: QueryN Metasearch +Disallow:/ +User-agent: Quick-Crawler +Disallow:/ +User-agent: RSSingBot +Disallow:/ +User-agent: RankActive +Disallow:/ +User-agent: RankActiveLinkBot +Disallow:/ +User-agent: RankFlex +Disallow:/ +User-agent: RankingBot +Disallow:/ +User-agent: RankingBot2 +Disallow:/ +User-agent: Rankivabot +Disallow:/ +User-agent: RankurBot +Disallow:/ +User-agent: Re-re +Disallow:/ +User-agent: ReGet +Disallow:/ +User-agent: RealDownload +Disallow:/ +User-agent: Reaper +Disallow:/ +User-agent: RebelMouse +Disallow:/ +User-agent: Recorder +Disallow:/ +User-agent: RedesScrapy +Disallow:/ +User-agent: RepoMonkey +Disallow:/ +User-agent: Ripper +Disallow:/ +User-agent: RocketCrawler +Disallow:/ +User-agent: Rogerbot +Disallow:/ +User-agent: SBIder +Disallow:/ +User-agent: SEOkicks +Disallow:/ +User-agent: SEOkicks-Robot +Disallow:/ +User-agent: SEOlyticsCrawler +Disallow:/ +User-agent: SEOprofiler +Disallow:/ +User-agent: SEOstats +Disallow:/ +User-agent: SISTRIX +Disallow:/ +User-agent: SMTBot +Disallow:/ +User-agent: SalesIntelligent +Disallow:/ +User-agent: ScanAlert +Disallow:/ +User-agent: Scanbot +Disallow:/ +User-agent: ScoutJet +Disallow:/ +User-agent: Scrapy +Disallow:/ +User-agent: Screaming +Disallow:/ +User-agent: ScreenerBot +Disallow:/ +User-agent: ScrepyBot +Disallow:/ +User-agent: Searchestate +Disallow:/ +User-agent: SearchmetricsBot +Disallow:/ +User-agent: Semrush +Disallow:/ +User-agent: SemrushBot +Disallow:/ +User-agent: SentiBot +Disallow:/ +User-agent: SeoSiteCheckup +Disallow:/ +User-agent: SeobilityBot +Disallow:/ +User-agent: Seomoz +Disallow:/ +User-agent: Shodan +Disallow:/ +User-agent: Siphon +Disallow:/ +User-agent: SiteCheckerBotCrawler +Disallow:/ +User-agent: SiteExplorer +Disallow:/ +User-agent: SiteLockSpider +Disallow:/ +User-agent: SiteSnagger +Disallow:/ +User-agent: SiteSucker +Disallow:/ +User-agent: Site Sucker +Disallow:/ +User-agent: Sitebeam +Disallow:/ +User-agent: Siteimprove +Disallow:/ +User-agent: Sitevigil +Disallow:/ +User-agent: SlySearch +Disallow:/ +User-agent: SmartDownload +Disallow:/ +User-agent: Snake +Disallow:/ +User-agent: Snapbot +Disallow:/ +User-agent: Snoopy +Disallow:/ +User-agent: SocialRankIOBot +Disallow:/ +User-agent: Sociscraper +Disallow:/ +User-agent: Sogou web spider +Disallow:/ +User-agent: Sosospider +Disallow:/ +User-agent: Sottopop +Disallow:/ +User-agent: SpaceBison +Disallow:/ +User-agent: Spammen +Disallow:/ +User-agent: SpankBot +Disallow:/ +User-agent: Spanner +Disallow:/ +User-agent: Spbot +Disallow:/ +User-agent: Spinn3r +Disallow:/ +User-agent: SputnikBot +Disallow:/ +User-agent: Sqlmap +Disallow:/ +User-agent: Sqlworm +Disallow:/ +User-agent: Sqworm +Disallow:/ +User-agent: Steeler +Disallow:/ +User-agent: Stripper +Disallow:/ +User-agent: Sucker +Disallow:/ +User-agent: Sucuri +Disallow:/ +User-agent: SuperBot +Disallow:/ +User-agent: SuperHTTP +Disallow:/ +User-agent: Surfbot +Disallow:/ +User-agent: SurveyBot +Disallow:/ +User-agent: Suzuran +Disallow:/ +User-agent: Swiftbot +Disallow:/ +User-agent: Szukacz +Disallow:/ +User-agent: T0PHackTeam +Disallow:/ +User-agent: T8Abot +Disallow:/ +User-agent: Teleport +Disallow:/ +User-agent: TeleportPro +Disallow:/ +User-agent: Telesoft +Disallow:/ +User-agent: Telesphoreo +Disallow:/ +User-agent: Telesphorep +Disallow:/ +User-agent: TheNomad +Disallow:/ +User-agent: The Intraformant +Disallow:/ +User-agent: Thumbor +Disallow:/ +User-agent: TightTwatBot +Disallow:/ +User-agent: Titan +Disallow:/ +User-agent: Toata +Disallow:/ +User-agent: Toweyabot +Disallow:/ +User-agent: Tracemyfile +Disallow:/ +User-agent: Trendiction +Disallow:/ +User-agent: Trendictionbot +Disallow:/ +User-agent: True_Robot +Disallow:/ +User-agent: Turingos +Disallow:/ +User-agent: Turnitin +Disallow:/ +User-agent: TurnitinBot +Disallow:/ +User-agent: TwengaBot +Disallow:/ +User-agent: Twice +Disallow:/ +User-agent: Typhoeus +Disallow:/ +User-agent: URLy.Warning +Disallow:/ +User-agent: URLy Warning +Disallow:/ +User-agent: UnisterBot +Disallow:/ +User-agent: Upflow +Disallow:/ +User-agent: V-BOT +Disallow:/ +User-agent: VB Project +Disallow:/ +User-agent: VCI +Disallow:/ +User-agent: Vacuum +Disallow:/ +User-agent: Vagabondo +Disallow:/ +User-agent: VelenPublicWebCrawler +Disallow:/ +User-agent: VeriCiteCrawler +Disallow:/ +User-agent: VidibleScraper +Disallow:/ +User-agent: Virusdie +Disallow:/ +User-agent: VoidEYE +Disallow:/ +User-agent: Voil +Disallow:/ +User-agent: Voltron +Disallow:/ +User-agent: WASALive-Bot +Disallow:/ +User-agent: WBSearchBot +Disallow:/ +User-agent: WEBDAV +Disallow:/ +User-agent: WISENutbot +Disallow:/ +User-agent: WPScan +Disallow:/ +User-agent: WWW-Collector-E +Disallow:/ +User-agent: WWW-Mechanize +Disallow:/ +User-agent: WWW::Mechanize +Disallow:/ +User-agent: WWWOFFLE +Disallow:/ +User-agent: Wallpapers/3.0 +Disallow:/ +User-agent: WallpapersHD +Disallow:/ +User-agent: WeSEE +Disallow:/ +User-agent: WebAuto +Disallow:/ +User-agent: WebBandit +Disallow:/ +User-agent: WebCollage +Disallow:/ +User-agent: WebCopier +Disallow:/ +User-agent: WebEnhancer +Disallow:/ +User-agent: WebFetch +Disallow:/ +User-agent: WebFuck +Disallow:/ +User-agent: WebGo IS +Disallow:/ +User-agent: WebImageCollector +Disallow:/ +User-agent: WebLeacher +Disallow:/ +User-agent: WebPix +Disallow:/ +User-agent: WebReaper +Disallow:/ +User-agent: WebSauger +Disallow:/ +User-agent: WebStripper +Disallow:/ +User-agent: WebSucker +Disallow:/ +User-agent: WebWhacker +Disallow:/ +User-agent: WebZIP +Disallow:/ +User-agent: Web Auto +Disallow:/ +User-agent: Web Collage +Disallow:/ +User-agent: Web Enhancer +Disallow:/ +User-agent: Web Fetch +Disallow:/ +User-agent: Web Fuck +Disallow:/ +User-agent: Web Pix +Disallow:/ +User-agent: Web Sauger +Disallow:/ +User-agent: Web Sucker +Disallow:/ +User-agent: Webalta +Disallow:/ +User-agent: WebmasterWorldForumBot +Disallow:/ +User-agent: Webshag +Disallow:/ +User-agent: WebsiteExtractor +Disallow:/ +User-agent: WebsiteQuester +Disallow:/ +User-agent: Website Quester +Disallow:/ +User-agent: Webster +Disallow:/ +User-agent: Whack +Disallow:/ +User-agent: Whacker +Disallow:/ +User-agent: Whatweb +Disallow:/ +User-agent: Who.is Bot +Disallow:/ +User-agent: Widow +Disallow:/ +User-agent: WinHTTrack +Disallow:/ +User-agent: WiseGuys Robot +Disallow:/ +User-agent: Wonderbot +Disallow:/ +User-agent: Woobot +Disallow:/ +User-agent: Wotbox +Disallow:/ +User-agent: Wprecon +Disallow:/ +User-agent: Xaldon WebSpider +Disallow:/ +User-agent: Xaldon_WebSpider +Disallow:/ +User-agent: Xenu +Disallow:/ +User-agent: YoudaoBot +Disallow:/ +User-agent: Zade +Disallow:/ +User-agent: Zauba +Disallow:/ +User-agent: Zermelo +Disallow:/ +User-agent: Zeus +Disallow:/ +User-agent: Zitebot +Disallow:/ +User-agent: ZmEu +Disallow:/ +User-agent: ZoomBot +Disallow:/ +User-agent: ZoominfoBot +Disallow:/ +User-agent: ZumBot +Disallow:/ +User-agent: ZyBorg +Disallow:/ +User-agent: archive.org_bot +Disallow:/ +User-agent: arquivo-web-crawler +Disallow:/ +User-agent: arquivo.pt +Disallow:/ +User-agent: autoemailspider +Disallow:/ +User-agent: backlink-check +Disallow:/ +User-agent: check1.exe +Disallow:/ +User-agent: coccocbot-web +Disallow:/ +User-agent: cognitiveseo +Disallow:/ +User-agent: com.plumanalytics +Disallow:/ +User-agent: crawl.sogou.com +Disallow:/ +User-agent: crawler.feedback +Disallow:/ +User-agent: crawler4j +Disallow:/ +User-agent: demandbase-bot +Disallow:/ +User-agent: domainsproject.org +Disallow:/ +User-agent: eCatch +Disallow:/ +User-agent: evc-batch +Disallow:/ +User-agent: facebookscraper +Disallow:/ +User-agent: gopher +Disallow:/ +User-agent: heritrix +Disallow:/ +User-agent: instabid +Disallow:/ +User-agent: internetVista monitor +Disallow:/ +User-agent: ips-agent +Disallow:/ +User-agent: isitwp.com +Disallow:/ +User-agent: lwp-request +Disallow:/ +User-agent: lwp-trivial +Disallow:/ +User-agent: magpie-crawler +Disallow:/ +User-agent: meanpathbot +Disallow:/ +User-agent: mediawords +Disallow:/ +User-agent: muhstik-scan +Disallow:/ +User-agent: netEstate NE Crawler +Disallow:/ +User-agent: oBot +Disallow:/ +User-agent: page scorer +Disallow:/ +User-agent: pcBrowser +Disallow:/ +User-agent: plumanalytics +Disallow:/ +User-agent: polaris version +Disallow:/ +User-agent: probe-image-size +Disallow:/ +User-agent: ripz +Disallow:/ +User-agent: s1z.ru +Disallow:/ +User-agent: satoristudio.net +Disallow:/ +User-agent: scalaj-http +Disallow:/ +User-agent: scan.lol +Disallow:/ +User-agent: seobility +Disallow:/ +User-agent: seoscanners +Disallow:/ +User-agent: seostar +Disallow:/ +User-agent: serpstatbot +Disallow:/ +User-agent: sexsearcher +Disallow:/ +User-agent: sitechecker.pro +Disallow:/ +User-agent: siteripz +Disallow:/ +User-agent: sogouspider +Disallow:/ +User-agent: sp_auditbot +Disallow:/ +User-agent: spyfu +Disallow:/ +User-agent: sysscan +Disallow:/ +User-agent: tAkeOut +Disallow:/ +User-agent: trendiction.com +Disallow:/ +User-agent: trendiction.de +Disallow:/ +User-agent: ubermetrics-technologies.com +Disallow:/ +User-agent: voyagerx.com +Disallow:/ +User-agent: webmeup-crawler +Disallow:/ +User-agent: x09Mozilla +Disallow:/ +User-agent: x22Mozilla +Disallow:/ +User-agent: xpymep1.exe +Disallow:/ +User-agent: zauba.io +Disallow:/ +User-agent: zgrab +Disallow:/ +